diff options
Diffstat (limited to 'src/main/java/com/mesasoft/cn/sketch')
12 files changed, 2042 insertions, 0 deletions
diff --git a/src/main/java/com/mesasoft/cn/sketch/api/BrightCloud.java b/src/main/java/com/mesasoft/cn/sketch/api/BrightCloud.java new file mode 100644 index 0000000..3a51220 --- /dev/null +++ b/src/main/java/com/mesasoft/cn/sketch/api/BrightCloud.java @@ -0,0 +1,193 @@ +package com.mesasoft.cn.sketch.api; + +import com.alibaba.fastjson.JSON; +import com.alibaba.fastjson.JSONArray; +import com.alibaba.fastjson.JSONObject; +import com.mesasoft.cn.sketch.config.ApplicationConfig; +import com.mesasoft.cn.sketch.entity.DomainCategory; +import com.mesasoft.cn.util.FileUtils; +import com.mesasoft.cn.util.ValidationUtils; +import org.apache.log4j.Logger; + +import java.io.*; +import java.net.HttpURLConnection; +import java.net.URL; +import java.nio.charset.StandardCharsets; +import java.util.*; + +/** + * @author yjy + * @version 1.0 + * @date 2021/2/22 2:37 下午 + */ + +public class BrightCloud { + private static final Logger LOG = Logger.getLogger(BrightCloud.class); + + private final Integer maxObjectNum = ApplicationConfig.API_BC_MAXIMUM_QUERYNUM; + private final HashMap<Integer, List<String>> catId2Info = new HashMap<>(); + private HttpURLConnection con; + + public List<DomainCategory> getQueryFiles(List<String> domains){ + JSONObject queryResults = getQueryResults(domains); + return responseSparse(queryResults); + } + + // 获取json格式查询结果 + public JSONObject getQueryResults(List<String> domains) { + if (domains.size()> ApplicationConfig.API_BC_MAXIMUM_QUERYNUM){ + LOG.warn("Too many domains in a http post request, the number of fqdn/url should be no more than " + + ApplicationConfig.API_BC_MAXIMUM_QUERYNUM + "!"); + } + JSONObject jsonRes = new JSONObject(); + try { + URL url = new URL(ApplicationConfig.API_BC_URL); + con = (HttpURLConnection) url.openConnection(); + con.setRequestMethod(ApplicationConfig.API_BC_METHOD); + con.setDoOutput(true); + con.setDoInput(true); + + con.setRequestProperty("Content-Type", "application/json"); + + JSONObject param = new JSONObject(); + param.put("oemid", ApplicationConfig.API_BC_OEMID); + param.put("deviceid", ApplicationConfig.API_BC_DEVICEID); + param.put("uid", ApplicationConfig.API_BC_UID); + + param.put("queries", Collections.singletonList(ApplicationConfig.API_BC_QUERYTYPE)); + param.put("a1cat", ApplicationConfig.API_BC_ISA1CAT); + param.put("reputation", ApplicationConfig.API_BC_ISREPU); + param.put("xml", ApplicationConfig.API_BC_ISXML); // json or xml格式 + + param.put("urls", domains); + + //建立实际的连接 + con.connect(); + OutputStreamWriter writer = new OutputStreamWriter(this.con.getOutputStream(), StandardCharsets.UTF_8); + writer.write(param.toString()); + writer.flush(); + } catch (IOException e) { + e.printStackTrace(); + } + + try { + // 获取服务端响应，通过输入流来读取URL的响应 + InputStream is = con.getInputStream(); + BufferedReader reader = new BufferedReader(new InputStreamReader(is, StandardCharsets.UTF_8)); + StringBuilder sbf = new StringBuilder(); + String strRead = null; + while ((strRead = reader.readLine()) != null) { + sbf.append(strRead); + sbf.append("\r\n"); + } + reader.close(); + + jsonRes = JSONObject.parseObject(sbf.toString()); + con.disconnect(); + } catch (IOException e) { + e.printStackTrace(); + } + return jsonRes; + } + + // json响应内容解析 + public List<DomainCategory> responseSparse(JSONObject records){ + List<DomainCategory> domainFiles = new ArrayList<>(); + Boolean querySucess = records.get("status").equals(200); + + if (!querySucess) { + System.out.print(records); + LOG.error("Wrong query. Query type: " + records.get("type")); + } else { + JSONArray array = records.getJSONArray("results"); + for (int i = 0; i < array.size(); i++) { + JSONObject jo = array.getJSONObject(i); + + // json处理 + JSONObject queries = jo.getJSONObject("queries"); + JSONObject getInfo = queries.getJSONObject(ApplicationConfig.API_BC_QUERYTYPE); + + JSONObject cat = getInfo.getJSONArray("cats").getJSONObject(0); + Integer catId = cat.getInteger("catid"); + String fqdn = jo.getString("url"); + domainFiles.add(new DomainCategory( + fqdn, + "brightcloud", + querySucess, + ValidationUtils.getMatchPattern(fqdn), + getInfo.getInteger("reputation"), + getRepLevel(getInfo.getInteger("reputation")), + catId, + getCatInfo(catId).get(0), + getCatInfo(catId).get(1), + cat.getInteger("conf"), + getInfo.getBoolean("a1cat"))); + } + } + return domainFiles; + } + + private String getRepLevel(Integer repScore){ + String level = null; //用str存放数据 + if (repScore > 80){ level="Trustworthy";} + else if (repScore > 60){ level="Low Risk";} + else if (repScore > 40){ level="Moderate Risk";} + else if (repScore > 20){ level="Suspicious";} + else if (repScore > 0){ level="High Risk";} + return level; + } + + // 获取类别id对应信息 + public void geneCatInfo(){ + if (catId2Info.size()==0){ + JSONObject jsonObject; +// String filePath = Objects.requireNonNull(BrightCloud.class.getClassLoader() +// .getResource(ApplicationConfig.API_BC_CATEINFO_FILE)).getFile(); + String filePath =ApplicationConfig.API_BC_CATEINFO_FILE; + String s = FileUtils.readJsonFile(filePath); + jsonObject = JSON.parseObject(s); + + if (!(jsonObject==null)){ + JSONObject tmp = (JSONObject) jsonObject.getJSONArray("results").get(0); + JSONArray catInfoArray = tmp.getJSONObject("queries").getJSONObject("getcatlist").getJSONArray("cats"); + + for (int i = 0; i < catInfoArray.size(); i++){ + JSONObject keyObject = catInfoArray.getJSONObject(i); + List<String> value = new ArrayList<>(Arrays.asList( + keyObject.getString("catname"), + keyObject.getString("catgroup"))); + catId2Info.put(i+1, value); + } + } + } + } + + public List<String> getCatInfo(Integer catId){ + List<String> info = Arrays.asList("", ""); + + if (0 < catId && catId <= 83) { + if (catId2Info.size()==0){ + geneCatInfo(); + } + + info = catId2Info.get(catId); + + if (info == null){ + LOG.error("Failed at geneCatInfo function"); + System.out.print("Failed at geneCatInfo function"); + } + } + + return info; + } + + public Integer getMaxObjectNum() { + return maxObjectNum; + } + + public static void main(String[] args) { + JSONObject queryResults = new BrightCloud().getQueryResults(Arrays.asList("baidu.com")); + new BrightCloud().responseSparse(queryResults); + } +} + diff --git a/src/main/java/com/mesasoft/cn/sketch/api/ChinaZ.java b/src/main/java/com/mesasoft/cn/sketch/api/ChinaZ.java new file mode 100644 index 0000000..1c75093 --- /dev/null +++ b/src/main/java/com/mesasoft/cn/sketch/api/ChinaZ.java @@ -0,0 +1,336 @@ +package com.mesasoft.cn.sketch.api; +/* + * @Description: + * @Author: chenxu + * @Date: 2021-12-27 13:59:29 + * @LastEditTime: 2021-12-29 17:05:45 + * @LastEditors: chenxu + */ + +import com.alibaba.fastjson.JSONArray; +import com.alibaba.fastjson.JSONObject; +import com.mesasoft.cn.sketch.config.ApplicationConfig; +import com.mesasoft.cn.sketch.entity.DomainWhois; +import org.apache.http.HttpResponse; +import org.apache.http.NameValuePair; +import org.apache.http.client.entity.UrlEncodedFormEntity; +import org.apache.http.client.methods.HttpPost; +import org.apache.http.impl.client.CloseableHttpClient; +import org.apache.http.impl.client.HttpClients; +import org.apache.http.message.BasicNameValuePair; +import org.apache.log4j.Logger; + +import java.io.BufferedReader; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.net.URI; +import java.nio.charset.StandardCharsets; +import java.sql.Date; +import java.util.*; +import java.util.concurrent.TimeUnit; + +public class ChinaZ { + private static final Logger LOG = Logger.getLogger(ChinaZ.class); + private final String apiKey = ApplicationConfig.API_CHINAZ_KEY; + + public List<DomainWhois> getQueryFiles(List<String> objectList) { + List<JSONObject> queryResults = getQueryResults(objectList); + return responseSparse(queryResults); + } + + /** + * @Description: 站长之家单查询 + * @Param : 域名 + * @Return: 查询结果 + */ + public JSONObject getQueryResult(String domain){ + String urlString = ApplicationConfig.API_CHINAZ_URL_SINGLE; + Map<String, String> params = new LinkedHashMap<String,String>(); + params.put("key", apiKey); + params.put("domain", domain); + return whoisInfoResolve(doPost(urlString, params),domain); + } + + /** + * @Description: 站长之家多域名查询 + * @Param : 域名 + * @Return: 查询结果 + */ + public List<JSONObject> getQueryResults(List<String> domainList){ + String urlString = ApplicationConfig.API_CHINAZ_URL_SINGLE; + List<JSONObject> whoisInfoList = new ArrayList<>(); + for (String s : domainList) { + Map<String, String> params = new LinkedHashMap<String, String>(); + params.put("key", apiKey); + params.put("domain", s); + JSONObject r = doPost(urlString, params); + whoisInfoList.add(whoisInfoResolve(r, s)); + } + return whoisInfoList; + } + + public List<DomainWhois> responseSparse(List<JSONObject> records){ + List<DomainWhois> whoisFiles = new ArrayList<>(); + + for(JSONObject record: records) { + Boolean querySucess = record.getBoolean("isSuccess"); + if (!querySucess) { + LOG.error("Failed query. Query response: " + record); + break; + } + + String fqdn = record.getString("domain_name"); + String domainName = record.getString("domain_host"); + Integer matchPattern = fqdn.equals(domainName)? 1 : 2 ; + String source = "chinaz"; + + // json处理 + Date creatDate = null; + Date expiraDate = null; + java.util.Date tmpDate = record.getDate("domain_whois_create_time"); + if(tmpDate!=null){ + creatDate = new Date(tmpDate.getTime()); + } + tmpDate = record.getDate("domain_whois_expiration_time"); + if(tmpDate!=null){ + expiraDate = new Date(tmpDate.getTime()); + } + whoisFiles.add(new DomainWhois( + fqdn, + source, + matchPattern, + record.getBoolean("isSuccess"), + record.getString("domain_host"), + null, + creatDate, + expiraDate, + record.getString("domain_whois_email"), + record.getString("domain_whois_name_servers"), + record.getString("domain_whois_registrar"), + null, + null, + null, + null, + null, + null, + null, + record.getString("domain_whois_phone") + )); + } + return whoisFiles; + } + /** + * @Description: 解析并重构JSON串 + * @Param : 查询得到的“单个”JSON串 + * @Return: 返回重构的JSON串 + */ + public JSONObject whoisInfoResolve(JSONObject jsonRes,String queryDomain){ + JSONObject whoisInfo = new JSONObject(true); + JSONObject res = jsonRes.getJSONObject("Result"); + if(jsonRes.get("StateCode").equals(1)){ + whoisInfo.put("isSuccess", jsonRes.get("StateCode")); + whoisInfo.put("domain_name",queryDomain); + whoisInfo.put("domain_host", res.get("Host")); + whoisInfo.put("domain_whois_create_time", res.get("CreationDate")); + whoisInfo.put("domain_whois_expiration_time", res.get("ExpirationDate")); + whoisInfo.put("domain_whois_registrar", res.get("Registrar")); + whoisInfo.put("whois_registrar_name", res.get("ContactPerson")); + whoisInfo.put("domain_whois_email", res.get("Email")); + whoisInfo.put("domain_whois_phone", res.get("Phone")); + whoisInfo.put("domain_whois_name_servers", res.get("DnsServer")); + whoisInfo.put("domain_whois_status", res.get("DomainStatus")); + }else{ + whoisInfo.put("isSuccess", jsonRes.get("StateCode")); + } + return whoisInfo; + } + + /** + * @Description: 构造批量查询需要的URL + * @Param : 待查询域名 + * @Return: 拼接好的URL + */ + public List<String> queryStringBuilder(List<String> domainList){ + //将域名每50个划分一组 + int CHINAZ_REQUEST_LIMIT = 50 ; + int domainListSize = domainList.size(); + int toIndex = CHINAZ_REQUEST_LIMIT; + Map domainListMap = new HashMap(); + int keyToken = 0; + for(int i = 0;i<domainList.size();i+=CHINAZ_REQUEST_LIMIT){ + if(i+CHINAZ_REQUEST_LIMIT>domainListSize){ //作用为toIndex最后没有50条数据则剩余几条newList中就装几条 + toIndex=domainListSize-i; + } + List<String> newList = domainList.subList(i,i+toIndex); + domainListMap.put("keyName"+keyToken, newList); + keyToken++; + } + //将批量查询的域名，构造成CHINAZ的格式 + List<String> domainListString = new ArrayList<>(); + Iterator iter = domainListMap.entrySet().iterator(); + while (iter.hasNext()) { + Map.Entry entry = (Map.Entry) iter.next(); + Object key = entry.getKey(); + Object val = entry.getValue(); + String urlString = ""; + urlString = String.valueOf(val); + urlString = urlString.replace(", ","|"); + urlString =urlString.replace("[","").replace("]",""); + domainListString.add(urlString); + } + return domainListString; + } + + /** + * @Description: 站长之家批量查询-批量提交查询请求，并获得提取任务ID + * @Param : 域名集合（不能超过50个） + * @Return: 查询结果 + */ + public String batchRequest_step1(String domainsString){ + String TaskID = ""; + String urlString = ApplicationConfig.API_CHINAZ_URL_BATCH; + + Map<String, String> params = new LinkedHashMap<String,String>(); + if (!Objects.equals(domainsString, "overflow")){ + params.put("domains",domainsString); + params.put("key", apiKey); + JSONObject r = doPost(urlString, params); + TaskID = r.get("TaskID").toString(); + return TaskID; + }else{ + return TaskID; + } + + } + + /** + * @Description: 站长之家查询-根据提取任务ID查询数据是否采集完成，如果完成则得到Json格式结果 + * @Param : 任务ID + * @Return: 查询结果 + */ + public JSONObject batchRequest_step2(String TaskID){ + String urlString = ApplicationConfig.API_CHINAZ_URL_BATCH; + + Map<String, String> params = new LinkedHashMap<String,String>(); + params.put("taskid",TaskID); + JSONObject requestTotal = null; + requestTotal = doPost(urlString, params); + return requestTotal; + } + + /** + * @Description: 完成如下内容：1）将domain拼接成50个一组；2）调用step_1的API上传数据；3）调用step_2的API获取数据；4）格式整理，输出数据 + * @Param : 域名列表 + * @Return: whois记录列表 + */ + public List<String> batchRequestController(List<String> domainList){ + + List<String> result = new ArrayList<>(); + if (domainList.size()> 5000){ + System.out.println("Too many urls in a http post request!"); + } + List<String> domainListString = new ArrayList<>(); + List<String> TaskID = new ArrayList<>(); + + + // Queue<String> queue = new LinkedList<String>(); + domainListString = queryStringBuilder(domainList); + //循环发送请求，收集每个请求的TaskID + for (String domainParam : domainListString) { + TaskID.add(batchRequest_step1(domainParam)); + } + + for (String s : TaskID) { + int flag = 0; + //查询接口数据，如果API仍在查询中，则等待10秒继续访问 + while (flag == 0) { + JSONObject data = batchRequest_step2(s); + if (data.get("StateCode").equals(0)) { + long timeToSleep = 10L; + TimeUnit time = TimeUnit.SECONDS; + try { + time.sleep(timeToSleep); + } catch (InterruptedException e) { + System.out.println("Interrupted " + "while Sleeping"); + } + } else { + flag = 1; + + JSONObject json_result = data.getJSONObject("Result"); + + JSONArray json_data = json_result.getJSONArray("Data"); + // //对Data内部的数据进行遍历 + for (int j = 0; j < json_data.size(); j++) { + String queryDomain = (String) json_data.getJSONObject(j).get("Domain"); + result.add(whoisInfoResolve(json_data.getJSONObject(j), queryDomain).toJSONString()); + } + } + } + } + + return result; + } + + /** + * @Description: POST调用API数据 + * @Param : 请求对URL，POST请求体需要添加的 k-v 数据 + * @Return: API JSON数据 + */ + public JSONObject doPost(String url, Map params){ + JSONObject jsonRes = null; + try { + // 定义HttpClient + CloseableHttpClient client = HttpClients.createDefault(); + // 实例化HTTP方法 + HttpPost request = new HttpPost(); + request.setURI(new URI(url)); + + //设置参数 + List<NameValuePair> nvps = new ArrayList<NameValuePair>(); + for (Object o : params.keySet()) { + String name = (String) o; + String value = String.valueOf(params.get(name)); + nvps.add(new BasicNameValuePair(name, value)); + + //System.out.println(name +"-"+value); + } + request.setEntity(new UrlEncodedFormEntity(nvps)); + //发送请求 + HttpResponse httpResponse = client.execute(request); + // 获取响应输入流 + InputStream inStream = httpResponse.getEntity().getContent(); + //对放回数据进行处理 + BufferedReader reader = new BufferedReader(new InputStreamReader(inStream , StandardCharsets.UTF_8)); + StringBuilder strber = new StringBuilder(); + StringBuilder sbf = new StringBuilder(); + String strRead = null; + while ((strRead = reader.readLine()) != null) { + sbf.append(strRead); + sbf.append("\r\n"); + } + // 关闭输入流 + inStream.close(); + jsonRes = JSONObject.parseObject(sbf.toString()); + }catch (Exception e) { + System.out.println("请求接口异常"); + } + return jsonRes; + } + + + public static void main(String[] args){ + ChinaZ t = new ChinaZ(); + + //单查询测试 +// System.out.println(t.singleRequest("aaa.baidu.com")); + + //批量查询测试 + List<String> domainList = new ArrayList<>(); + domainList.add("www.baidu.com"); +// domainList.add("aaa.qq.com"); +// domainList.add("doc.mesalab.com"); + +// System.out.println(t.batchRequestController(domainList)); + System.out.println(t.getQueryResults(domainList)); + } +} diff --git a/src/main/java/com/mesasoft/cn/sketch/config/ApplicationConfig.java b/src/main/java/com/mesasoft/cn/sketch/config/ApplicationConfig.java new file mode 100644 index 0000000..099205e --- /dev/null +++ b/src/main/java/com/mesasoft/cn/sketch/config/ApplicationConfig.java @@ -0,0 +1,54 @@ +package com.mesasoft.cn.sketch.config; + + +import com.mesasoft.cn.util.ConfigUtils; + +public class ApplicationConfig { + public static final String QUERY_OUTPUT_DIR = ConfigUtils.getStringProperty("query.output.dir"); + + public static final String QUERY_TYPES_DOMAIN = ConfigUtils.getStringProperty("query.types.domain"); + public static final String QUERY_TYPES_IP = ConfigUtils.getStringProperty("query.types.ip"); + + public static final Integer UPDATE_EXPIRED_DAY = ConfigUtils.getIntProperty("update.expired.day"); // 更新任务中过期时间长度（天数） + + public static final Integer QUERY_READIN_BATCH = ConfigUtils.getIntProperty("query.readin.batch"); + public static final Integer QUERY_LOG_FILE_LINE_INTERVAL = ConfigUtils.getIntProperty("query.log.file.line.interval"); // 文件查询时，打印log的读取行数间隔 + + // api参数 + // brightcloud + public static final String API_BC_OEMID = ConfigUtils.getStringProperty("bc.oemid"); + public static final String API_BC_DEVICEID = ConfigUtils.getStringProperty("bc.deviceid"); + public static final String API_BC_UID = ConfigUtils.getStringProperty("bc.uid"); + public static final String API_BC_URL = ConfigUtils.getStringProperty("bc.url"); + public static final String API_BC_METHOD = ConfigUtils.getStringProperty("bc.method"); + + public static final String API_BC_ISA1CAT = ConfigUtils.getStringProperty("bc.isa1cat"); + public static final String API_BC_ISREPU = ConfigUtils.getStringProperty("bc.isReputation"); + public static final String API_BC_ISXML = ConfigUtils.getStringProperty("bc.isxml"); + + + public static final Integer API_BC_MAXIMUM_QUERYNUM = ConfigUtils.getIntProperty("bc.maximum.query.num"); // brightcloud单次查询条数上线 + public static final String API_BC_QUERYTYPE = ConfigUtils.getStringProperty("bc.queryType"); + + public static final String API_BC_USE_REPORT_FILE = ConfigUtils.getStringProperty("bc.usereport.filepath"); // brightcloud使用报告导出文件目录 + public static final String API_BC_CATEINFO_FILE = ConfigUtils.getStringProperty("bc.cateinfo.filepath"); // brightcloud使用报告导出文件目录 + + + // chinaz + public static final String API_CHINAZ_URL_SINGLE = ConfigUtils.getStringProperty("chinaz.url.single"); + public static final String API_CHINAZ_URL_BATCH = ConfigUtils.getStringProperty("chinaz.url.batch"); + public static final String API_CHINAZ_KEY = ConfigUtils.getStringProperty("chinaz.key"); + public static final Integer API_CHINAZ_MAXIMUM_QUERYNUM = ConfigUtils.getIntProperty("chinaz.maximum.query.num"); + public static final String API_CHINAZ_USE_REPORT_FILE = ConfigUtils.getStringProperty("chinaz.usereport.filepath"); + + // Mariadb + public static final String DATABASE = ConfigUtils.getStringProperty("database"); + public static final String DOMAIN_CATE_TABLENAME = ConfigUtils.getStringProperty("tablename.domain.category"); + public static final String DOMAIN_WHOIS_TABLENAME = ConfigUtils.getStringProperty("tablename.domain.whois"); + public static final String DNS_SERVER_TABLENAME = ConfigUtils.getStringProperty("tablename.dns.server"); + public static final Integer DB_QUERY_BATCH_SIZE = ConfigUtils.getIntProperty("db.query.batch.size"); + + // 其他 + public static final String TLD_FILE = ConfigUtils.getStringProperty("tld.file"); // 顶级域名公开列表文件 + +} diff --git a/src/main/java/com/mesasoft/cn/sketch/config/MariaDbBase.java b/src/main/java/com/mesasoft/cn/sketch/config/MariaDbBase.java new file mode 100644 index 0000000..ba617ce --- /dev/null +++ b/src/main/java/com/mesasoft/cn/sketch/config/MariaDbBase.java @@ -0,0 +1,84 @@ +package com.mesasoft.cn.sketch.config; + +import com.mesasoft.cn.util.TimeUtils; +import org.apache.log4j.Logger; + +import java.sql.*; +import java.util.Date; +import java.util.Properties; + +/** + * Created with IntelliJ IDEA. + * User: joy + * Date: 2021/12/28 + * Time: 2:56 PM + * Description: No Description + */ +public class MariaDbBase { + + private static final Logger LOG = Logger.getLogger(MariaDbBase.class); + private static final Properties props = new Properties(); + + private final Statement statement; + + public MariaDbBase(Connection conn, Statement stat) { + statement = stat; + } + + /** + * 执行写入sql + */ + public void writeSqlExecute(String sql){ + try { + statement.executeUpdate(sql); + } catch (SQLIntegrityConstraintViolationException e){ + LOG.error("Duplicated entry for key 'PRIMARY'"); + } catch (SQLException exception) { + LOG.error("Sql : " + sql); + exception.printStackTrace(); + } + } + + /** + * 执行查询sql + */ + public ResultSet querySqlExecute(String sql){ + ResultSet set = null; + try { + set = statement.executeQuery(sql); + } catch (SQLException exception) { + exception.printStackTrace(); + } + return set; + } + + + /** + * 获得指定表格、按指定时间字段的过期记录 + * @param tableName 库表名称 + * @param timeColumnName 时间列名 + * @return 查询结果 + */ + public ResultSet getExpiredRecord(String tableName, String timeColumnName){ + Date lastUpdateTime = new Timestamp(getExpiredTime(ApplicationConfig.UPDATE_EXPIRED_DAY).getTime()); + + String resSql = "SELECT *" + + " FROM " + ApplicationConfig.DATABASE + "." + tableName + + " WHERE " + timeColumnName + " < '" + lastUpdateTime + '\''; + + LOG.debug("Update task: expired query sql" + resSql); + + return querySqlExecute(resSql); + } + + /** + * TODO: getUnlabeledRecord() 考虑多个来源的情况 + */ + + /** + * 获得过期时间, 当前时间的expiredRangeDays天之前的日期为过期日期 + */ + public static Date getExpiredTime(int expiredRangeDays){ + return new Timestamp(TimeUtils.getStartOfDay(-expiredRangeDays).getTime()); + } +} diff --git a/src/main/java/com/mesasoft/cn/sketch/config/SketchDatabaseConfig.java b/src/main/java/com/mesasoft/cn/sketch/config/SketchDatabaseConfig.java new file mode 100644 index 0000000..dc69bbd --- /dev/null +++ b/src/main/java/com/mesasoft/cn/sketch/config/SketchDatabaseConfig.java @@ -0,0 +1,125 @@ +//package com.zhazhapan.efo.sketch.config; +// +//import com.alibaba.druid.pool.DruidDataSource; +//import lombok.Data; +//import org.apache.ibatis.session.SqlSessionFactory; +//import org.mybatis.spring.SqlSessionFactoryBean; +//import org.mybatis.spring.annotation.MapperScan; +//import org.springframework.beans.factory.annotation.Qualifier; +//import org.springframework.beans.factory.annotation.Value; +//import org.springframework.boot.context.properties.ConfigurationProperties; +//import org.springframework.context.annotation.Bean; +//import org.springframework.context.annotation.Configuration; +//import org.springframework.core.io.support.PathMatchingResourcePatternResolver; +//import org.springframework.jdbc.datasource.DataSourceTransactionManager; +// +//import javax.sql.DataSource; +//import java.sql.SQLException; +// +///** +// * @ProjectName +// * @Description: 后台数据源配置类 +// */ +//@Data +//@Configuration +//@ConfigurationProperties(prefix = "sketch.datasource.druid") +//@MapperScan(basePackages = SketchDatabaseConfig.PACKAGE, sqlSessionFactoryRef = "sketchSqlSessionFactory") +//public class SketchDatabaseConfig { +// /** +// * dao层的包路径 +// */ +// static final String PACKAGE = "com.mao.mysqlhive.demomh.mapper.sketch"; +// +// /** +// * mapper文件的相对路径 +// */ +// private static final String MAPPER_LOCATION = "classpath:mappers/sketch/*Mapper.xml"; +// +// @Value("${sketch.datasource.druid.filters}") +// private String filters; +// @Value("${sketch.datasource.druid.driverClassName}") +// private String url; +// @Value("${sketch.datasource.druid.url}") +// private String username; +// @Value("${sketch.datasource.druid.username}") +// private String password; +// @Value("${sketch.datasource.druid.password}") +// private String driverClassName; +// @Value("${sketch.datasource.druid.initialSize}") +// private int initialSize; +// @Value("${sketch.datasource.druid.minIdle}") +// private int minIdle; +// @Value("${sketch.datasource.druid.maxActive}") +// private int maxActive; +// @Value("${sketch.datasource.druid.maxWait}") +// private long maxWait; +// @Value("${sketch.datasource.druid.timeBetweenEvictionRunsMillis}") +// private long timeBetweenEvictionRunsMillis; +// @Value("${sketch.datasource.druid.minEvictableIdleTimeMillis}") +// private long minEvictableIdleTimeMillis; +// @Value("${sketch.datasource.druid.validationQuery}") +// private String validationQuery; +// @Value("${sketch.datasource.druid.testWhileIdle}") +// private boolean testWhileIdle; +// @Value("${sketch.datasource.druid.testOnBorrow}") +// private boolean testOnBorrow; +// @Value("${sketch.datasource.druid.testOnReturn}") +// private boolean testOnReturn; +// @Value("${sketch.datasource.druid.poolPreparedStatements}") +// private boolean poolPreparedStatements; +// @Value("${sketch.datasource.druid.maxPoolPreparedStatementPerConnectionSize}") +// private int maxPoolPreparedStatementPerConnectionSize; +// +// +// @Bean(name = "sketchDataSource") +// public DataSource sketchDataSource() throws SQLException { +// DruidDataSource druid = new DruidDataSource(); +// // 监控统计拦截的filters +// druid.setFilters(filters); +// +// // 配置基本属性 +// druid.setDriverClassName(driverClassName); +// druid.setUsername(username); +// druid.setPassword(password); +// druid.setUrl(url); +// +// //初始化时建立物理连接的个数 +// druid.setInitialSize(initialSize); +// //最大连接池数量 +// druid.setMaxActive(maxActive); +// //最小连接池数量 +// druid.setMinIdle(minIdle); +// //获取连接时最大等待时间，单位毫秒。 +// druid.setMaxWait(maxWait); +// //间隔多久进行一次检测，检测需要关闭的空闲连接 +// druid.setTimeBetweenEvictionRunsMillis(timeBetweenEvictionRunsMillis); +// //一个连接在池中最小生存的时间 +// druid.setMinEvictableIdleTimeMillis(minEvictableIdleTimeMillis); +// //用来检测连接是否有效的sql +// druid.setValidationQuery(validationQuery); +// //建议配置为true，不影响性能，并且保证安全性。 +// druid.setTestWhileIdle(testWhileIdle); +// //申请连接时执行validationQuery检测连接是否有效 +// druid.setTestOnBorrow(testOnBorrow); +// druid.setTestOnReturn(testOnReturn); +// //是否缓存preparedStatement，也就是PSCache，oracle设为true，mysql设为false。分库分表较多推荐设置为false +// druid.setPoolPreparedStatements(poolPreparedStatements); +// // 打开PSCache时，指定每个连接上PSCache的大小 +// druid.setMaxPoolPreparedStatementPerConnectionSize(maxPoolPreparedStatementPerConnectionSize); +// return druid; +// } +// +// @Bean(name = "sketchTransactionManager") +// public DataSourceTransactionManager sketchTransactionManager() throws SQLException { +// return new DataSourceTransactionManager(sketchDataSource()); +// } +// +// @Bean(name = "sketchSqlSessionFactory") +// public SqlSessionFactory sketchSqlSessionFactory(@Qualifier("sketchDataSource") DataSource sketchDataSource) throws Exception { +// final SqlSessionFactoryBean sessionFactory = new SqlSessionFactoryBean(); +// sessionFactory.setDataSource(sketchDataSource); +// sessionFactory.setMapperLocations(new PathMatchingResourcePatternResolver().getResources(SketchDatabaseConfig.MAPPER_LOCATION)); +// +// return sessionFactory.getObject(); +// } +//} diff --git a/src/main/java/com/mesasoft/cn/sketch/controller/SketchDomainController.java b/src/main/java/com/mesasoft/cn/sketch/controller/SketchDomainController.java new file mode 100644 index 0000000..736a5be --- /dev/null +++ b/src/main/java/com/mesasoft/cn/sketch/controller/SketchDomainController.java @@ -0,0 +1,54 @@ +package com.mesasoft.cn.sketch.controller; + +import com.alibaba.fastjson.JSONObject; +import com.mesasoft.cn.annotation.AuthInterceptor; +import com.mesasoft.cn.enums.InterceptorLevel; +import com.mesasoft.cn.service.IFileManagerService; +import io.swagger.annotations.Api; +import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RequestMethod; +import org.springframework.web.bind.annotation.RestController; + +import javax.servlet.http.HttpServletRequest; + +/** + * @description: + * @author: zhq + * @create: 2022-03-10 + **/ +@RestController +@RequestMapping("/sketch/domain") +@Api(value = "/sketch/domain", description = "文件相关操作") +@Slf4j +public class SketchDomainController { + + private final IFileManagerService fileManagerService; + private final HttpServletRequest request; + private JSONObject jsonObject; + + @Value("${sketch.path.admin}") + private String pathAdmin; + @Value("${sketch.path.user}") + private String pathUser; + + @Autowired + public SketchDomainController(HttpServletRequest request, IFileManagerService fileManagerService, JSONObject jsonObject) { + this.fileManagerService = fileManagerService; + this.jsonObject = jsonObject; + this.request = request; + } + + @AuthInterceptor(InterceptorLevel.USER) + @RequestMapping(value = "/list", method = RequestMethod.GET) + public String list(String searchPath) { + + log.info("search path :", searchPath); + + + return jsonObject.toJSONString(); + } + +} diff --git a/src/main/java/com/mesasoft/cn/sketch/controller/SketchFileController.java b/src/main/java/com/mesasoft/cn/sketch/controller/SketchFileController.java new file mode 100644 index 0000000..b9c9ba1 --- /dev/null +++ b/src/main/java/com/mesasoft/cn/sketch/controller/SketchFileController.java @@ -0,0 +1,93 @@ +package com.mesasoft.cn.sketch.controller; + +import com.alibaba.fastjson.JSONObject; +import com.mesasoft.cn.annotation.AuthInterceptor; +import com.mesasoft.cn.entity.User; +import com.mesasoft.cn.enums.InterceptorLevel; +import com.mesasoft.cn.service.IFileManagerService; +import com.mesasoft.cn.util.ControllerUtils; +import com.zhazhapan.modules.constant.ValueConsts; +import com.zhazhapan.util.ArrayUtils; +import io.swagger.annotations.Api; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang3.StringUtils; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RequestMethod; +import org.springframework.web.bind.annotation.RestController; +import org.springframework.web.multipart.MultipartFile; +import org.springframework.web.multipart.MultipartHttpServletRequest; + +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; +import java.io.IOException; +import java.util.Map; + +/** + * @description: + * @author: zhq + * @create: 2022-03-10 + **/ +@RestController +@RequestMapping("/sketch") +@Api(value = "/sketch", description = "文件相关操作") +@Slf4j +public class SketchFileController { + + private final IFileManagerService fileManagerService; + private final HttpServletRequest request; + private JSONObject jsonObject; + + @Value("${sketch.path.admin}") + private String pathAdmin; + @Value("${sketch.path.user}") + private String pathUser; + + @Autowired + public SketchFileController(HttpServletRequest request, IFileManagerService fileManagerService, JSONObject jsonObject) { + this.fileManagerService = fileManagerService; + this.jsonObject = jsonObject; + this.request = request; + } + + @AuthInterceptor(InterceptorLevel.USER) + @RequestMapping(value = "/list", method = RequestMethod.GET) + public String list(String searchPath) { + + log.info("search path :", searchPath); + + User user = (User) request.getSession().getAttribute(ValueConsts.USER_STRING); + JSONObject json = new JSONObject(); + String rootPath = ""; + if ("system".equals(user.getUsername())) { + rootPath = pathAdmin; + } else { + rootPath = pathUser; + } + searchPath = searchPath.replace("\\\\", ""); + searchPath = searchPath.replace("//", ""); + if (StringUtils.isNotBlank(searchPath) && !rootPath.contains(searchPath)) { + json.fluentPut("path", searchPath); + } else { + json.fluentPut("path", rootPath); + } + //返回结果 + jsonObject.put("path", json.getString("path")); + jsonObject.put("result", fileManagerService.list(json)); + return jsonObject.toJSONString(); + } + + @RequestMapping(value = "/upload", method = RequestMethod.POST) + public String upload(String destination, MultipartHttpServletRequest request) { + Map<String, MultipartFile> fileMap = request.getFileMap(); + MultipartFile[] files = ArrayUtils.mapToArray(fileMap, MultipartFile.class); + jsonObject.put("result", fileManagerService.upload(destination, files)); + return jsonObject.toJSONString(); + } + + @RequestMapping(value = "/download", method = RequestMethod.GET) + public void download(HttpServletRequest request, HttpServletResponse response, String path) throws IOException, ClassNotFoundException { + ControllerUtils.loadResource2(response, path, ValueConsts.TRUE); + } +} diff --git a/src/main/java/com/mesasoft/cn/sketch/dao/SketchDAO.java b/src/main/java/com/mesasoft/cn/sketch/dao/SketchDAO.java new file mode 100644 index 0000000..0473403 --- /dev/null +++ b/src/main/java/com/mesasoft/cn/sketch/dao/SketchDAO.java @@ -0,0 +1,81 @@ +package com.mesasoft.cn.sketch.dao; + +import com.mesasoft.cn.sketch.dao.sqlprovider.SketchSqlProvider; +import com.mesasoft.cn.sketch.entity.DomainCategory; +import com.mesasoft.cn.dao.sqlprovider.UserSqlProvider; +import org.apache.ibatis.annotations.*; +import org.springframework.stereotype.Repository; + +import java.util.List; + +@Repository +public interface SketchDAO { + + /** + * 通过域名查询 + */ + @Select("select * from domain_category_reputation where fqdn=#{fqdn}") + DomainCategory getDomainCategoryByFqdn(String usernameOrEmail); + + /** + * 获取列表 + * + * @return {@link List} + */ + @SelectProvider(type = SketchSqlProvider.class, method = "getDomainCategoryListBy") + List<DomainCategory> getDomainCategoryList(@Param("condition") String condition, @Param("offset") int offset); + + + /** + * 添加一个用户 + * + * @param domainCategory {@link DomainCategory} + * @return 是否插入成功 + */ + @Insert("insert into domain_category_reputation(username,real_name,email,password,is_downloadable,is_uploadable,is_deletable," + + "is_updatable,is_visible) values(#{username},#{realName},#{email},sha2(#{password},256)," + + "#{isDownloadable},#{isUploadable},#{isDeletable},#{isUpdatable},#{isVisible})") + boolean insertDomainCategory(DomainCategory domainCategory); + + /** + * 通过ID更新用户基本信息 + * + * @param id 编号 + * @param avatar 头像 + * @param realName 真实姓名 + * @param email 邮箱 + * @return 是否更新成功 + */ + @Update("update user set avatar=#{avatar},real_name=#{realName},email=#{email} where id=#{id}") + boolean updateBasicInfo(@Param("id") int id, @Param("avatar") String avatar, @Param("realName") String realName, + @Param("email") String email); + + + + /** + * 通过id更新用户登录时间 + * + * @param id 编号 + * @return {@link Boolean} + */ + @Update("update user set last_login_time=current_timestamp where id=#{id}") + boolean updateUserLoginTime(int id); + + /** + * 更新操作用户权限 + * + * @param id 用户编号 + * @param isDownloadable 下载权限 + * @param isUploadable 上传权限 + * @param isVisible 可查权限 + * @param isDeletable 删除权限 + * @param isUpdatable 更新权限 + * @return {@link Boolean} + */ + @UpdateProvider(type = UserSqlProvider.class, method = "updateAuthById") + boolean updateAuthById(@Param("id") int id, @Param("isDownloadable") int isDownloadable, + @Param("isUploadable") int isUploadable, @Param("isDeletable") int isDeletable, @Param( + "isUpdatable") int isUpdatable, @Param("isVisible") int isVisible); + + +} diff --git a/src/main/java/com/mesasoft/cn/sketch/dao/sqlprovider/SketchSqlProvider.java b/src/main/java/com/mesasoft/cn/sketch/dao/sqlprovider/SketchSqlProvider.java new file mode 100644 index 0000000..53e24a2 --- /dev/null +++ b/src/main/java/com/mesasoft/cn/sketch/dao/sqlprovider/SketchSqlProvider.java @@ -0,0 +1,27 @@ +package com.mesasoft.cn.sketch.dao.sqlprovider; + +import com.mesasoft.cn.SketchApplication; +import com.mesasoft.cn.modules.constant.ConfigConsts; +import com.zhazhapan.util.Checker; +import org.apache.ibatis.annotations.Param; +import org.apache.ibatis.jdbc.SQL; + +/** + * @author pantao + * @since 2018/1/19 + */ +public class SketchSqlProvider { + + public String getDomainCategoryListBy(@Param("condition") String condition, @Param("offset") int offset) { + String sql = new SQL() {{ + SELECT("*"); + FROM("domain_category_reputation"); + if (Checker.isNotEmpty(condition)) { + WHERE("fqdn like '%" + condition + "'"); + } + ORDER_BY(SketchApplication.settings.getStringUseEval(ConfigConsts.USER_ORDER_BY_OF_SETTINGS)); + }}.toString(); + int size = SketchApplication.settings.getIntegerUseEval(ConfigConsts.USER_PAGE_SIZE_OF_SETTINGS); + return sql + " limit " + (offset * size) + "," + size; + } +} diff --git a/src/main/java/com/mesasoft/cn/sketch/entity/DomainCategory.java b/src/main/java/com/mesasoft/cn/sketch/entity/DomainCategory.java new file mode 100644 index 0000000..dbcfdc1 --- /dev/null +++ b/src/main/java/com/mesasoft/cn/sketch/entity/DomainCategory.java @@ -0,0 +1,340 @@ +package com.mesasoft.cn.sketch.entity; + +import com.alibaba.fastjson.JSONObject; +import com.mesasoft.cn.util.ConfigUtils; +import com.mesasoft.cn.sketch.config.ApplicationConfig; +import com.mesasoft.cn.util.MariaDbBase; +import com.mesasoft.cn.util.ValidationUtils; + +import java.sql.ResultSet; +import java.sql.SQLException; +import java.util.ArrayList; +import java.util.List; +import java.util.stream.Collectors; + +/** + * Created with IntelliJ IDEA. + * User: joy + * Date: 2021/12/29 + * Time: 9:27 AM + * Description: No Description + */ +public class DomainCategory { + private static final String dataBase = ApplicationConfig.DATABASE; + private static final String tableName = ApplicationConfig.DOMAIN_CATE_TABLENAME; + + private String fqdn; + private String source; + private Boolean query_success; + private Integer match_pattern; + private Integer reputation_score; + private String reputation_level; + private Integer category_id; + private String category_name; + private String category_group; + private Integer category_conf; + private Boolean is_a1_cat; + private Integer status_code = 0; + private String submit_user ="''"; + + + // category schema + public DomainCategory(String fqdn, + String source, + Boolean query_success, + Integer match_pattern, + Integer reputation_score, + String reputationLevel, + Integer categoryId, + String categoryName, + String categoryGroup, + Integer categoryConf, + Boolean isA1Cat, Integer statusCode, String submitUser) { + + this.fqdn = fqdn; + this.source = source; // 默认应为为brightcloud + this.query_success = query_success; + + // 没有设置match_pattern，则二级域名为右匹配，其余为全匹配 + if (match_pattern == null) { + this.match_pattern = ValidationUtils.getMatchPattern(fqdn); + } else { + this.match_pattern = match_pattern; + } + + this.reputation_score = reputation_score; + this.reputation_level = ConfigUtils.getEffectiveString(reputationLevel); + this.category_id = categoryId; + this.category_name = ConfigUtils.getEffectiveString(categoryName); + this.category_group = ConfigUtils.getEffectiveString(categoryGroup); + this.category_conf = categoryConf; + this.is_a1_cat = isA1Cat; + this.status_code = statusCode; + this.submit_user = submitUser; + + } + public DomainCategory(String fqdn, + String source, + Boolean query_success, + Integer match_pattern, + Integer reputation_score, + String reputationLevel, + Integer categoryId, + String categoryName, + String categoryGroup, + Integer categoryConf, + Boolean isA1Cat) { + + this.fqdn = fqdn; + this.source = source; // 默认应为为brightcloud + this.query_success = query_success; + + // 没有设置match_pattern，则二级域名为右匹配，其余为全匹配 + if (match_pattern == null) { + this.match_pattern = ValidationUtils.getMatchPattern(fqdn); + } else { + this.match_pattern = match_pattern; + } + + this.reputation_score = reputation_score; + this.reputation_level = ConfigUtils.getEffectiveString(reputationLevel); + this.category_id = categoryId; + this.category_name = ConfigUtils.getEffectiveString(categoryName); + this.category_group = ConfigUtils.getEffectiveString(categoryGroup); + this.category_conf = categoryConf; + this.is_a1_cat = isA1Cat; + + } + public static void insertRecords(List<DomainCategory> categoryFiles, MariaDbBase mariaDbBase) { + for (DomainCategory categoryFile : categoryFiles) { + // 生成sql + String resSql = "INSERT INTO " + dataBase + "." + tableName + ' ' + + " (" + categoryFile.getKeys() + ") values" + + '(' + categoryFile.getValues() + ')'; + resSql = resSql.replace("'null'", "null"); + + mariaDbBase.writeSqlExecute(resSql); + } + } + + public void updateRecords(List<DomainCategory> categoryFiles, MariaDbBase mariaDbBase) { + for (DomainCategory categoryFile : categoryFiles) { + + String resSql = "UPDATE " + dataBase + "." + + tableName + ' ' + + "SET " + categoryFile.getKeyValues() + + ", update_time = current_time() " + + " WHERE fqdn = '" + categoryFile.getFqdn() + '\''; + resSql = resSql.replace("'null'", "null"); + + mariaDbBase.writeSqlExecute(resSql); + } + } + + public static List<DomainCategory> getDbRecord(List<String> fqdns, MariaDbBase mariaDbBase, String source) throws SQLException { + String queryFqdns = fqdns.stream().map(s -> "'" + s + "'").collect(Collectors.joining(",")); + String sql = "SELECT * FROM " + dataBase + "." + + tableName + ' ' + + " WHERE fqdn in (" + queryFqdns + ") and source = '" + source + "'"; + + return rs2schema(mariaDbBase.querySqlExecute(sql)); + } + + public static List<DomainCategory> rs2schema(ResultSet rs) throws SQLException { + List<DomainCategory> schemaFiles = new ArrayList<>(); + while (rs.next()) { + schemaFiles.add( + new DomainCategory( + rs.getString("fqdn"), + rs.getString("source"), + rs.getBoolean("query_success"), + rs.getInt("match_pattern"), + rs.getInt("reputation_score"), + rs.getString("reputation_level"), + rs.getInt("category_id"), + rs.getString("category_name"), + rs.getString("category_group"), + rs.getInt("category_conf"), + rs.getBoolean("is_a1_cat"), + rs.getInt("status_code"), + rs.getString("submit_user") + + )); + } + return schemaFiles; + } + + public static JSONObject schema2json(DomainCategory schema) throws SQLException { + JSONObject jsonObject = new JSONObject(true); + jsonObject.put("fqdn", schema.getFqdn()); + jsonObject.put("source", schema.getSource()); + jsonObject.put("query_success", schema.getQuery_success()); + jsonObject.put("match_pattern", schema.getMatch_pattern()); + jsonObject.put("reputation_score", schema.getReputation_score()); + jsonObject.put("reputation_level", schema.getReputation_level()); + jsonObject.put("category_id", schema.getCategory_id()); + jsonObject.put("category_group", schema.getCategory_group()); + jsonObject.put("category_name", schema.getCategory_name()); + jsonObject.put("category_conf", schema.getCategory_conf()); + jsonObject.put("is_a1_cat", schema.getIs_a1_cat()); + jsonObject.put("status_code", schema.getStatus_code()); + jsonObject.put("submit_user", schema.getSubmit_user()); + + return jsonObject; + } + + public String getValues() { + String resString = "'" + fqdn + '\'' + + ", '" + source + '\'' + + ", " + query_success + + ", " + match_pattern + + ", " + reputation_score + + ", '" + reputation_level + '\'' + + ", " + category_id + + ", '" + category_name + '\'' + + ", '" + category_group + '\'' + + ", " + category_conf + + ", " + status_code + + ", " + submit_user + + ", " + is_a1_cat; + + return resString.replace("'null'", "null"); + } + + public String getKeys() { + String resString; + resString = "fqdn" + + ", source" + + ", query_success" + + ", match_pattern" + + ", reputation_score" + + ", reputation_level" + + ", category_id" + + ", category_name" + + ", category_group" + + ", category_conf" + + ", status_code" + + ", submit_user" + + ", is_a1_cat"; + return resString; + } + + public String getKeyValues() { + String resString = "source='" + source + '\'' + + ", query_success=" + query_success + + ", match_pattern=" + match_pattern + + ", reputation_score=" + reputation_score + + ", reputation_level='" + reputation_level + '\'' + + ", category_id=" + category_id + + ", category_name='" + category_name + '\'' + + ", category_group='" + category_group + '\'' + + ", category_conf=" + category_conf + + ", is_a1_cat=" + is_a1_cat; + + return resString.replace("'null'", "null"); + } + + public String getFqdn() { + return fqdn; + } + + public void setFqdn(String fqdn) { + this.fqdn = fqdn; + } + + public String getSource() { + return source; + } + + public void setSource(String source) { + this.source = source; + } + + public Boolean getQuery_success() { + return query_success; + } + + public void setQuery_success(Boolean query_success) { + this.query_success = query_success; + } + + public Integer getMatch_pattern() { + return match_pattern; + } + + public void setMatch_pattern(Integer match_pattern) { + this.match_pattern = match_pattern; + } + + public Integer getReputation_score() { + return reputation_score; + } + + public void setReputation_score(Integer reputation_score) { + this.reputation_score = reputation_score; + } + + public String getReputation_level() { + return reputation_level; + } + + public void setReputation_level(String reputation_level) { + this.reputation_level = reputation_level; + } + + public Integer getCategory_id() { + return category_id; + } + + public void setCategory_id(Integer category_id) { + this.category_id = category_id; + } + + public String getCategory_name() { + return category_name; + } + + public void setCategory_name(String category_name) { + this.category_name = category_name; + } + + public String getCategory_group() { + return category_group; + } + + public void setCategory_group(String category_group) { + this.category_group = category_group; + } + + public Integer getCategory_conf() { + return category_conf; + } + + public void setCategory_conf(Integer category_conf) { + this.category_conf = category_conf; + } + + public Boolean getIs_a1_cat() { + return is_a1_cat; + } + + public void setIs_a1_cat(Boolean is_a1_cat) { + this.is_a1_cat = is_a1_cat; + } + + public Integer getStatus_code() { + return status_code; + } + + public void setStatus_code(Integer status_code) { + this.status_code = status_code; + } + + public String getSubmit_user() { + return submit_user; + } + + public void setSubmit_user(String submit_user) { + this.submit_user = submit_user; + } +} diff --git a/src/main/java/com/mesasoft/cn/sketch/entity/DomainWhois.java b/src/main/java/com/mesasoft/cn/sketch/entity/DomainWhois.java new file mode 100644 index 0000000..55679a8 --- /dev/null +++ b/src/main/java/com/mesasoft/cn/sketch/entity/DomainWhois.java @@ -0,0 +1,423 @@ +package com.mesasoft.cn.sketch.entity; + +import com.alibaba.fastjson.JSONObject; +import com.mesasoft.cn.sketch.config.ApplicationConfig; +import com.mesasoft.cn.util.ConfigUtils; +import com.mesasoft.cn.util.MariaDbBase; +import com.mesasoft.cn.util.ValidationUtils; + +import java.sql.Date; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.sql.Timestamp; +import java.util.ArrayList; +import java.util.List; +import java.util.stream.Collectors; + +/** + * Created with IntelliJ IDEA. + * User: joy + * Date: 2021/12/29 + * Time: 9:27 AM + * Description: No Description + */ +public class DomainWhois { + private static final String dataBase = ApplicationConfig.DATABASE; + private static final String tableName = ApplicationConfig.DOMAIN_WHOIS_TABLENAME; + + private String fqdn; + private String source; + private Boolean query_success; + private Integer match_pattern; + private String whois_domain; + private Timestamp whois_update_date; + private Timestamp whois_create_date; + private Timestamp whois_expire_date; + private String whois_email; + private String whois_ns; + private String whois_registrar_name; + private String whois_registrant_org; + private String whois_registrant_name; + private String whois_registrant_street; + private String whois_registrant_city; + private String whois_registrant_state; + private String whois_registrant_postcode; + private String whois_registrant_country; + private String whois_registrant_phone; + + + // category schema + public DomainWhois(String fqdn, + String source, + Integer match_pattern, + Boolean query_success, + String whoisDomain, + Date whoisUpdateDate, + Date whoisCreateDate, + Date whoisExpireDate, + String whoisEmail, + String whoisNs, + String whoisRegistrarName, + String whoisRegistrantOrg, + String whoisRegistrantName, + String whoisRegistrantStreet, + String whoisRegistrantCity, + String whoisRegistrantState, + String whoisRegistrantPostcode, + String whoisRegistrantCountry, + String whoisRegistrantPhone + ) { + + this.fqdn = fqdn; + this.source = source; + + // 没有设置match_pattern，则二级域名为右匹配，其余为全匹配 + if (match_pattern == null) { + this.match_pattern = ValidationUtils.getMatchPattern(fqdn); + } else { + this.match_pattern = match_pattern; + } + + this.query_success = query_success; + this.whois_domain = ConfigUtils.getEffectiveString(whoisDomain); + this.whois_update_date = whoisUpdateDate == null ? null : new Timestamp(whoisUpdateDate.getTime()); + this.whois_create_date = whoisCreateDate == null ? null : new Timestamp(whoisCreateDate.getTime()); + this.whois_expire_date = whoisExpireDate == null ? null : new Timestamp(whoisExpireDate.getTime()); + this.whois_email = ConfigUtils.getEffectiveString(whoisEmail); + this.whois_ns = ConfigUtils.getEffectiveString(whoisNs); + this.whois_registrar_name = ConfigUtils.getEffectiveString(whoisRegistrarName); + this.whois_registrant_org = ConfigUtils.getEffectiveString(whoisRegistrantOrg); + this.whois_registrant_name = ConfigUtils.getEffectiveString(whoisRegistrantName); + this.whois_registrant_street = ConfigUtils.getEffectiveString(whoisRegistrantStreet); + this.whois_registrant_city = ConfigUtils.getEffectiveString(whoisRegistrantCity); + this.whois_registrant_state = ConfigUtils.getEffectiveString(whoisRegistrantState); + this.whois_registrant_postcode = ConfigUtils.getEffectiveString(whoisRegistrantPostcode); + this.whois_registrant_country = ConfigUtils.getEffectiveString(whoisRegistrantCountry); + this.whois_registrant_phone = ConfigUtils.getEffectiveString(whoisRegistrantPhone); + } + + public static void insertRecords(List<DomainWhois> whoisFiles, MariaDbBase mariaDbBase) { + for (DomainWhois whoisFile : whoisFiles) { + // 生成sql + String resSql = "INSERT INTO " + dataBase + "." + tableName + ' ' + + " (" + whoisFile.getKeys() + ") values" + + '(' + whoisFile.getValues() + ')'; + resSql = resSql.replace("'null'", "null"); + + mariaDbBase.writeSqlExecute(resSql); + } + } + + public static String insertSql(List<DomainWhois> whoisFiles) { + DomainWhois whoisFile = whoisFiles.get(0); + // 生成sql + String resSql = "INSERT INTO " + dataBase + "." + tableName + ' ' + + " (" + whoisFile.getKeys() + ") values" + + '(' + whoisFile.getValues() + ')'; + resSql = resSql.replace("'null'", "null"); + + return resSql; + } + + public static void updateRecords(List<DomainWhois> categoryFiles, MariaDbBase mariaDbBase) { + for (DomainWhois categoryFile : categoryFiles) { + + String resSql = "UPDATE " + dataBase + "." + + tableName + ' ' + + "SET " + categoryFile.getKeyValues() + + ", update_time = current_time() " + + " WHERE fqdn = '" + categoryFile.getFqdn() + '\''; + resSql = resSql.replace("'null'", "null"); + + mariaDbBase.writeSqlExecute(resSql); + } + } + + public static List<DomainWhois> getDbRecord(List<String> fqdns, MariaDbBase mariaDbBase, String source) throws SQLException { + String queryFqdns = fqdns.stream().map(s -> "'" + s + "'").collect(Collectors.joining(",")); + String sql = "SELECT * FROM " + dataBase + "." + + tableName + ' ' + + " WHERE fqdn in (" + queryFqdns + ") "; + + return rs2schema(mariaDbBase.querySqlExecute(sql)); + } + + public String getValues() { + String resString = "'" + fqdn + '\'' + + ", '" + source + '\'' + + ", " + query_success + + ", " + match_pattern + + ", '" + whois_domain + '\'' + + ", '" + whois_update_date + '\'' + + ", '" + whois_create_date + '\'' + + ", '" + whois_expire_date + '\'' + + ", '" + whois_email + '\'' + + ", '" + whois_ns + '\'' + + ", '" + whois_registrar_name + '\'' + + ", '" + whois_registrant_org + '\'' + + ", '" + whois_registrant_name + '\'' + + ", '" + whois_registrant_street + '\'' + + ", '" + whois_registrant_city + '\'' + + ", '" + whois_registrant_state + '\'' + + ", '" + whois_registrant_postcode + '\'' + + ", '" + whois_registrant_country + '\'' + + ", '" + whois_registrant_phone + '\''; + + return resString.replace("'null'", "null"); + } + + public static List<DomainWhois> rs2schema(ResultSet rs) throws SQLException { + List<DomainWhois> schemaFiles = new ArrayList<>(); + while (rs.next()) { + schemaFiles.add( + new DomainWhois( + rs.getString("fqdn"), + rs.getString("source"), + rs.getInt("match_pattern"), + rs.getBoolean("query_success"), + rs.getString("whois_domain"), + (Date) rs.getDate("whois_update_date"), + (Date) rs.getDate("whois_create_date"), + (Date) rs.getDate("whois_expire_date"), + rs.getString("whois_email"), + rs.getString("whois_ns"), + rs.getString("whois_registrar_name"), + rs.getString("whois_registrant_org"), + rs.getString("whois_registrant_name"), + rs.getString("whois_registrant_street"), + rs.getString("whois_registrant_city"), + rs.getString("whois_registrant_state"), + rs.getString("whois_registrant_postcode"), + rs.getString("whois_registrant_country"), + rs.getString("whois_registrant_phone") + )); + } + return schemaFiles; + } + + public static JSONObject schema2json(DomainWhois schema) throws SQLException { + JSONObject jsonObject = new JSONObject(true); + jsonObject.put("fqdn", schema.getFqdn()); + jsonObject.put("source", schema.getSource()); + jsonObject.put("match_pattern", schema.getMatch_pattern()); + jsonObject.put("query_success", schema.getQuery_success()); + jsonObject.put("whois_domain", schema.getWhois_domain()); + jsonObject.put("whois_update_date", schema.getWhois_update_date()); + jsonObject.put("whois_create_date", schema.getWhois_create_date()); + jsonObject.put("whois_expire_date", schema.getWhois_expire_date()); + jsonObject.put("whois_email", schema.getWhois_email()); + jsonObject.put("whois_ns", schema.getWhois_ns()); + jsonObject.put("whois_registrar_name", schema.getWhois_registrar_name()); + jsonObject.put("whois_registrant_org", schema.getWhois_registrant_org()); + jsonObject.put("whois_registrant_name", schema.getWhois_registrant_name()); + jsonObject.put("whois_registrant_street", schema.getWhois_registrant_street()); + jsonObject.put("whois_registrant_city", schema.getWhois_registrant_city()); + jsonObject.put("whois_registrant_state", schema.getWhois_registrant_state()); + jsonObject.put("whois_registrant_postcode", schema.getWhois_registrant_postcode()); + jsonObject.put("whois_registrant_country", schema.getWhois_registrant_country()); + jsonObject.put("whois_registrant_phone", schema.getWhois_registrant_phone()); + + return jsonObject; + } + + public String getKeys() { + String resString; + resString = "fqdn" + + ", source" + + ", query_success" + + ", match_pattern" + + ", whois_domain" + + ", whois_update_date" + + ", whois_create_date" + + ", whois_expire_date" + + ", whois_email" + + ", whois_ns" + + ", whois_registrar_name" + + ", whois_registrant_org" + + ", whois_registrant_name" + + ", whois_registrant_street" + + ", whois_registrant_city" + + ", whois_registrant_state" + + ", whois_registrant_postcode" + + ", whois_registrant_country" + + ", whois_registrant_phone"; + return resString; + } + + public String getKeyValues() { + String resString = "query_success=" + query_success + + ", source='" + source + '\'' + + ", match_pattern=" + match_pattern + + ", whois_domain='" + whois_domain + '\'' + + ", whois_update_date='" + whois_update_date + '\'' + + ", whois_create_date='" + whois_create_date + '\'' + + ", whois_expire_date='" + whois_expire_date + '\'' + + ", whois_email='" + whois_email + '\'' + + ", whois_ns='" + whois_ns + '\'' + + ", whois_registrar_name='" + whois_registrar_name + '\'' + + ", whois_registrant_org='" + whois_registrant_org + '\'' + + ", whois_registrant_name='" + whois_registrant_name + '\'' + + ", whois_registrant_street='" + whois_registrant_street + '\'' + + ", whois_registrant_city='" + whois_registrant_city + '\'' + + ", whois_registrant_state='" + whois_registrant_state + '\'' + + ", whois_registrant_postcode='" + whois_registrant_postcode + '\'' + + ", whois_registrant_country='" + whois_registrant_country + '\'' + + ", whois_registrant_phone='" + whois_registrant_phone + '\''; + + return resString.replace("'null'", "null"); + } + + + public String getSource() { + return source; + } + + public void setSource(String source) { + this.source = source; + } + + public String getFqdn() { + return fqdn; + } + + public void setFqdn(String fqdn) { + this.fqdn = fqdn; + } + + public Boolean getQuery_success() { + return query_success; + } + + public void setQuery_success(Boolean query_success) { + this.query_success = query_success; + } + + public Integer getMatch_pattern() { + return match_pattern; + } + + public void setMatch_pattern(Integer match_pattern) { + this.match_pattern = match_pattern; + } + + public String getWhois_domain() { + return whois_domain; + } + + public void setWhois_domain(String whois_domain) { + this.whois_domain = whois_domain; + } + + public Timestamp getWhois_update_date() { + return whois_update_date; + } + + public void setWhois_update_date(Timestamp whois_update_date) { + this.whois_update_date = whois_update_date; + } + + public Timestamp getWhois_create_date() { + return whois_create_date; + } + + public void setWhois_create_date(Timestamp whois_create_date) { + this.whois_create_date = whois_create_date; + } + + public Timestamp getWhois_expire_date() { + return whois_expire_date; + } + + public void setWhois_expire_date(Timestamp whois_expire_date) { + this.whois_expire_date = whois_expire_date; + } + + public String getWhois_email() { + return whois_email; + } + + public void setWhois_email(String whois_email) { + this.whois_email = whois_email; + } + + public String getWhois_ns() { + return whois_ns; + } + + public void setWhois_ns(String whois_ns) { + this.whois_ns = whois_ns; + } + + public String getWhois_registrar_name() { + return whois_registrar_name; + } + + public void setWhois_registrar_name(String whois_registrar_name) { + this.whois_registrar_name = whois_registrar_name; + } + + public String getWhois_registrant_org() { + return whois_registrant_org; + } + + public void setWhois_registrant_org(String whois_registrant_org) { + this.whois_registrant_org = whois_registrant_org; + } + + public String getWhois_registrant_name() { + return whois_registrant_name; + } + + public void setWhois_registrant_name(String whois_registrant_name) { + this.whois_registrant_name = whois_registrant_name; + } + + public String getWhois_registrant_street() { + return whois_registrant_street; + } + + public void setWhois_registrant_street(String whois_registrant_street) { + this.whois_registrant_street = whois_registrant_street; + } + + public String getWhois_registrant_city() { + return whois_registrant_city; + } + + public void setWhois_registrant_city(String whois_registrant_city) { + this.whois_registrant_city = whois_registrant_city; + } + + public String getWhois_registrant_state() { + return whois_registrant_state; + } + + public void setWhois_registrant_state(String whois_registrant_state) { + this.whois_registrant_state = whois_registrant_state; + } + + public String getWhois_registrant_postcode() { + return whois_registrant_postcode; + } + + public void setWhois_registrant_postcode(String whois_registrant_postcode) { + this.whois_registrant_postcode = whois_registrant_postcode; + } + + public String getWhois_registrant_country() { + return whois_registrant_country; + } + + public void setWhois_registrant_country(String whois_registrant_country) { + this.whois_registrant_country = whois_registrant_country; + } + + public String getWhois_registrant_phone() { + return whois_registrant_phone; + } + + public void setWhois_registrant_phone(String whois_registrant_phone) { + this.whois_registrant_phone = whois_registrant_phone; + } +} + diff --git a/src/main/java/com/mesasoft/cn/sketch/service/SketchService.java b/src/main/java/com/mesasoft/cn/sketch/service/SketchService.java new file mode 100644 index 0000000..bb15d13 --- /dev/null +++ b/src/main/java/com/mesasoft/cn/sketch/service/SketchService.java @@ -0,0 +1,232 @@ +/* +package com.zhazhapan.efo.sketch.service; + +import cn.ac.iie.api.ChinaZ; +import cn.ac.iie.dao.MariaDbBase; +import cn.ac.iie.util.MariaDBUtils; +import cn.ac.iie.util.ValidationUtils; +import com.alibaba.fastjson.JSONArray; +import com.alibaba.fastjson.JSONObject; +import com.google.common.collect.Lists; +import com.zhazhapan.efo.sketch.api.BrightCloud; +import com.zhazhapan.efo.sketch.api.ChinaZ; +import com.zhazhapan.efo.sketch.config.ApplicationConfig; +import com.zhazhapan.efo.sketch.entity.DomainCategory; +import com.zhazhapan.efo.sketch.entity.DomainWhois; +import com.zhazhapan.efo.util.FileUtils; +import org.apache.log4j.Logger; + +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.OutputStream; +import java.io.OutputStreamWriter; +import java.nio.charset.StandardCharsets; +import java.sql.Connection; +import java.sql.Date; +import java.sql.SQLException; +import java.sql.Statement; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.stream.Collectors; + +*/ +/** + * Created with IntelliJ IDEA. + * User: joy + * Date: 2021/12/31 + * Time: 11:28 AM + * Description: No Description + *//* + +public class SketchService { + private static final Logger LOG = Logger.getLogger(SketchService.class); + private static final int MAX_DB_BATCH_SIZE = ApplicationConfig.DB_QUERY_BATCH_SIZE; + + private List<String> queryObjects; + private String curQueryType; + + private MariaDbBase mariaDB; + + long queryNum; + long dbResultNum; + long apiResultNum; + long failedQueryNum; + + public JSONArray getQueryResults(String objectType, String queryType, List<String> queryObjects, String username, Boolean isLocal) throws SQLException, IOException { + queryNum = 0; + dbResultNum = 0; + apiResultNum = 0; + failedQueryNum = 0; + + this.queryObjects = queryObjects; + this.curQueryType = queryType; + JSONArray json = null; + + Connection mariaConn = MariaDBUtils.getConnection(); + Statement mariaStat = mariaConn.createStatement(); + mariaDB = new MariaDbBase(mariaConn, mariaStat); + + //去重 + queryObjects = queryObjects.stream().distinct().collect(Collectors.toList()); + + // 校验 + queryObjects = ValidationUtils.getChecked(queryObjects, objectType); + this.queryNum = queryObjects.size(); + + // 执行查询 + if (queryType.equals("domain_category")) { + json = getDomainCategory(queryObjects, username, isLocal); + } else if (queryType.equals("domain_whois")) { + json = getDomainWhois(queryObjects, isLocal); + } else { + // TODO： get dns server info + LOG.error("Wrong query type: " + queryType); + } + + LOG.info("Query Statistic - number of query objects: " + queryNum + + "\nresults from local db: " + dbResultNum + + " results from external api: " + apiResultNum); + MariaDBUtils.close(mariaStat, mariaConn); + return json; + } + + public JSONArray getDomainCategory(List<String> domains, String username, boolean isLocal) throws SQLException, IOException { + + JSONArray results = new JSONArray(); + + // 查询本地数据库 + ArrayList<String> objectsFromDB = new ArrayList<>(); + List<List<String>> partitions = Lists.partition(queryObjects, MAX_DB_BATCH_SIZE); + for (List<String> partition : partitions) { // 批量查询 + List<DomainCategory> dbRecords = DomainCategory.getDbRecord(partition, mariaDB, "brightcloud"); + for (DomainCategory record : dbRecords) { + objectsFromDB.add(record.getFqdn()); // 保存查询记录 + JSONObject jsonObject = DomainCategory.schema2json(record); + results.add(jsonObject); // 保存查询结果 + } + } + dbResultNum = results.size(); + + // 调用api + List<DomainCategory> bcResults = new ArrayList<>(); + List<String> objectsFromApi = new ArrayList<>(queryObjects); + objectsFromApi.removeAll(objectsFromDB); + if (!isLocal && objectsFromApi.size() > 0) { + BrightCloud brightCloud = new BrightCloud(); + + List<List<String>> apiPartitions = Lists.partition(objectsFromApi, ApplicationConfig.API_BC_MAXIMUM_QUERYNUM); + for (List<String> partition : apiPartitions) { // 批量查询 + List<DomainCategory> recordsFromBcApi = brightCloud.responseSparse(brightCloud.getQueryResults(partition)); + for (DomainCategory record : recordsFromBcApi) { + if (record.getQuery_success().equals(true)) { //查询成功的结果 + record.setSubmit_user(username); + bcResults.add(record); + if (bcResults.size() > MAX_DB_BATCH_SIZE) { //超过一定量时写入数据库 + DomainCategory.insertRecords(bcResults, mariaDB); + bcResults.clear(); + } + results.add(DomainCategory.schema2json(record)); + } else { // 查询失败的结果 + failedQueryNum += 1; + } + } + } + } + apiResultNum = results.size() - dbResultNum - failedQueryNum; + DomainCategory.insertRecords(bcResults, mariaDB); + + if (apiResultNum > 0) { // 记录api调用次数 + OutputStream bcQueryLogStream = new FileOutputStream(ApplicationConfig.API_BC_USE_REPORT_FILE, true); + OutputStreamWriter bcQueryLogWriter = new OutputStreamWriter(bcQueryLogStream, StandardCharsets.UTF_8); + Date d = new Date(System.currentTimeMillis()); + bcQueryLogWriter.write(d + "," + "List Query," + "-" + "," + curQueryType + "," + apiResultNum + "\n"); + FileUtils.writerClose(bcQueryLogWriter, bcQueryLogStream); + } + + return results; + } + + public JSONArray getDomainWhois(List<String> domains, boolean isLocal) throws SQLException, IOException { + + JSONArray results = new JSONArray(); + + // 查询本地数据库 + ArrayList<String> objectsFromDB = new ArrayList<>(); + List<List<String>> partitions = Lists.partition(queryObjects, MAX_DB_BATCH_SIZE); + for (List<String> partition : partitions) { // 批量查询 + List<DomainWhois> dbRecords = + DomainWhois.getDbRecord(partition, mariaDB, "chinaz"); + for (DomainWhois record : dbRecords) { + objectsFromDB.add(record.getFqdn()); // 保存查询记录 + JSONObject jsonObject = DomainWhois.schema2json(record); + results.add(jsonObject); // 保存查询结果 + } + } + dbResultNum = results.size(); + + // 调用api + List<DomainWhois> chinazResults = new ArrayList<>(); + if (!isLocal) { + ChinaZ chinaz = new ChinaZ(); + + List<String> objectsFromApi = new ArrayList<>(queryObjects); + objectsFromApi.removeAll(objectsFromDB); // 需要调用api查询的部分对象 + + List<List<String>> apiPartitions = Lists.partition(objectsFromApi, ApplicationConfig.API_CHINAZ_MAXIMUM_QUERYNUM); + for (List<String> partition : apiPartitions) { // 批量查询 + List<DomainWhois> recordsFromApi = chinaz.responseSparse(chinaz.getQueryResults(partition)); + for (DomainWhois record : recordsFromApi) { + if (record.getQuery_success().equals(true)) { // 查询成功的结果 + chinazResults.add(record); + if (chinazResults.size() > MAX_DB_BATCH_SIZE) { // 超过一定量时写入数据库 + DomainWhois.insertRecords(chinazResults, mariaDB); + chinazResults.clear(); + } + results.add(DomainWhois.schema2json(record)); + } else { // 查询失败的结果 + failedQueryNum += 1; + } + } + } + } + + DomainWhois.insertRecords(chinazResults, mariaDB); + apiResultNum = results.size() - dbResultNum - failedQueryNum; + // todo 不直接写入，维护变量，导出一次 + if (apiResultNum > 0) { // 记录api调用次数 + OutputStream bcQueryLogStream = new FileOutputStream(ApplicationConfig.API_CHINAZ_USE_REPORT_FILE, true); + OutputStreamWriter bcQueryLogWriter = new OutputStreamWriter(bcQueryLogStream, StandardCharsets.UTF_8); + Date d = new Date(System.currentTimeMillis()); + bcQueryLogWriter.write(d + "," + "List Query," + "-" + "," + curQueryType + "," + apiResultNum + "\n"); + FileUtils.writerClose(bcQueryLogWriter, bcQueryLogStream); + } + + + return results; + } + + + public static void main(String[] args) throws Exception { +// String objectType = args[0]; +// String queryType = args[1]; +// List<String> queryObjects = Arrays.asList(args[2].split(",")); +// String username = args[4]; +// boolean isLocal = true; +// if (args.length >= 3) { +// isLocal = Boolean.parseBoolean(args[3]); +// } + +// System.out.println(new SketchService().getQueryResults(objectType, queryType, queryObjects, username, isLocal).toString()); + + JSONArray queryResults = new SketchService().getQueryResults("domain", + "domain_category", + Arrays.asList("baidu.com", "cctv.com"), "", true); + System.err.println(queryResults.toJSONString()); + String json = "{\"namespace\":\"log.session\",\"type\":\"record\",\"name\":\"session\",\"fields\":[{\"name\":\"common_log_id\",\"type\":\"long\"},{\"name\":\"common_service\",\"type\":\"long\"},{\"name\":\"common_recv_time\",\"type\":\"long\"},{\"name\":\"common_direction\",\"type\":\"long\"},{\"name\":\"common_l4_protocol\",\"type\":\"string\"},{\"name\":\"common_address_type\",\"type\":\"long\"},{\"name\":\"common_schema_type\",\"type\":\"string\"},{\"name\":\"common_policy_id\",\"type\":\"long\"},{\"name\":\"common_user_tags\",\"type\":\"string\"},{\"name\":\"common_action\",\"type\":\"long\"},{\"name\":\"common_sub_action\",\"type\":\"string\"},{\"name\":\"common_user_region\",\"type\":\"string\"},{\"name\":\"common_client_ip\",\"type\":\"string\"},{\"name\":\"common_client_port\",\"type\":\"long\"},{\"name\":\"common_internal_ip\",\"type\":\"string\"},{\"name\":\"common_entrance_id\",\"type\":\"long\"},{\"name\":\"common_device_id\",\"type\":\"string\"},{\"name\":\"common_egress_link_id\",\"type\":\"long\"},{\"name\":\"common_ingress_link_id\",\"type\":\"long\"},{\"name\":\"common_isp\",\"type\":\"string\"},{\"name\":\"common_device_tag\",\"type\":\"string\"},{\"name\":\"common_data_center\",\"type\":\"string\"},{\"name\":\"common_encapsulation\",\"type\":\"long\"},{\"name\":\"common_tunnels\",\"type\":\"string\"},{\"name\":\"common_sled_ip\",\"type\":\"string\"},{\"name\":\"common_device_group\",\"type\":\"string\"},{\"name\":\"common_app_behavior\",\"type\":\"string\"},{\"name\":\"common_client_location\",\"type\":\"string\"},{\"name\":\"common_client_asn\",\"type\":\"string\"},{\"name\":\"common_subscriber_id\",\"type\":\"string\"},{\"name\":\"common_imei\",\"type\":\"string\"},{\"name\":\"common_imsi\",\"type\":\"string\"},{\"name\":\"common_phone_number\",\"type\":\"string\"},{\"name\":\"common_server_ip\",\"type\":\"string\"},{\"name\":\"common_server_port\",\"type\":\"long\"},{\"name\":\"common_external_ip\",\"type\":\"string\"},{\"name\":\"common_server_location\",\"type\":\"string\"},{\"name\":\"common_server_asn\",\"type\":\"string\"},{\"name\":\"common_protocol_label\",\"type\":\"string\"},{\"name\":\"common_service_category\",\"type\":{\"type\":\"array\",\"items\":\"long\"}},{\"name\":\"common_app_label\",\"type\":\"string\"},{\"name\":\"common_app_id\",\"type\":\"string\"},{\"name\":\"common_userdefine_app_name\",\"type\":\"string\"},{\"name\":\"common_app_surrogate_id\",\"type\":\"string\"},{\"name\":\"common_l7_protocol\",\"type\":\"string\"},{\"name\":\"common_sessions\",\"type\":\"long\"},{\"name\":\"common_c2s_pkt_num\",\"type\":\"long\"},{\"name\":\"common_s2c_pkt_num\",\"type\":\"long\"},{\"name\":\"common_c2s_pkt_diff\",\"type\":\"long\"},{\"name\":\"common_s2c_pkt_diff\",\"type\":\"long\"},{\"name\":\"common_c2s_byte_diff\",\"type\":\"long\"},{\"name\":\"common_s2c_byte_diff\",\"type\":\"long\"},{\"name\":\"common_c2s_byte_num\",\"type\":\"long\"},{\"name\":\"common_s2c_byte_num\",\"type\":\"long\"},{\"name\":\"common_start_time\",\"type\":\"long\"},{\"name\":\"common_end_time\",\"type\":\"long\"},{\"name\":\"common_establish_latency_ms\",\"type\":\"long\"},{\"name\":\"common_con_duration_ms\",\"type\":\"long\"},{\"name\":\"common_stream_dir\",\"type\":\"long\"},{\"name\":\"common_address_list\",\"type\":\"string\"},{\"name\":\"common_has_dup_traffic\",\"type\":\"long\"},{\"name\":\"common_stream_error\",\"type\":\"string\"},{\"name\":\"common_stream_trace_id\",\"type\":\"long\"},{\"name\":\"common_link_info_c2s\",\"type\":\"string\"},{\"name\":\"common_link_info_s2c\",\"type\":\"string\"},{\"name\":\"common_packet_capture_file\",\"type\":\"string\"},{\"name\":\"common_c2s_ipfrag_num\",\"type\":\"long\"},{\"name\":\"common_s2c_ipfrag_num\",\"type\":\"long\"},{\"name\":\"common_c2s_tcp_lostlen\",\"type\":\"long\"},{\"name\":\"common_s2c_tcp_lostlen\",\"type\":\"long\"},{\"name\":\"common_c2s_tcp_unorder_num\",\"type\":\"long\"},{\"name\":\"common_s2c_tcp_unorder_num\",\"type\":\"long\"},{\"name\":\"common_c2s_pkt_retrans\",\"type\":\"long\"},{\"name\":\"common_s2c_pkt_retrans\",\"type\":\"long\"},{\"name\":\"common_c2s_byte_retrans\",\"type\":\"long\"},{\"name\":\"common_s2c_byte_retrans\",\"type\":\"long\"},{\"name\":\"common_tcp_client_isn\",\"type\":\"long\"},{\"name\":\"common_tcp_server_isn\",\"type\":\"long\"},{\"name\":\"common_mirrored_pkts\",\"type\":\"long\"},{\"name\":\"common_mirrored_bytes\",\"type\":\"long\"},{\"name\":\"common_first_ttl\",\"type\":\"long\"},{\"name\":\"common_processing_time\",\"type\":\"long\"},{\"name\":\"http_url\",\"type\":\"string\"},{\"name\":\"http_host\",\"type\":\"string\"},{\"name\":\"http_domain\",\"type\":\"string\"},{\"name\":\"http_request_line\",\"type\":\"string\"},{\"name\":\"http_response_line\",\"type\":\"string\"},{\"name\":\"http_request_header\",\"type\":\"string\"},{\"name\":\"http_response_header\",\"type\":\"string\"},{\"name\":\"http_request_content\",\"type\":\"string\"},{\"name\":\"http_response_content\",\"type\":\"string\"},{\"name\":\"http_request_body\",\"type\":\"string\"},{\"name\":\"http_response_body\",\"type\":\"string\"},{\"name\":\"http_request_body_key\",\"type\":\"string\"},{\"name\":\"http_response_body_key\",\"type\":\"string\"},{\"name\":\"http_proxy_flag\",\"type\":\"long\"},{\"name\":\"http_sequence\",\"type\":\"long\"},{\"name\":\"http_snapshot\",\"type\":\"string\"},{\"name\":\"http_cookie\",\"type\":\"string\"},{\"name\":\"http_referer\",\"type\":\"string\"},{\"name\":\"http_user_agent\",\"type\":\"string\"},{\"name\":\"http_request_content_length\",\"type\":\"string\"},{\"name\":\"http_request_content_type\",\"type\":\"string\"},{\"name\":\"http_response_content_length\",\"type\":\"string\"},{\"name\":\"http_response_content_type\",\"type\":\"string\"},{\"name\":\"http_content_length\",\"type\":\"string\"},{\"name\":\"http_content_type\",\"type\":\"string\"},{\"name\":\"http_set_cookie\",\"type\":\"string\"},{\"name\":\"http_version\",\"type\":\"string\"},{\"name\":\"http_response_latency_ms\",\"type\":\"long\"},{\"name\":\"http_session_duration_ms\",\"type\":\"long\"},{\"name\":\"http_action_file_size\",\"type\":\"long\"},{\"name\":\"mail_protocol_type\",\"type\":\"string\"},{\"name\":\"mail_account\",\"type\":\"string\"},{\"name\":\"mail_to_cmd\",\"type\":\"string\"},{\"name\":\"mail_from_cmd\",\"type\":\"string\"},{\"name\":\"mail_from\",\"type\":\"string\"},{\"name\":\"mail_to\",\"type\":\"string\"},{\"name\":\"mail_cc\",\"type\":\"string\"},{\"name\":\"mail_bcc\",\"type\":\"string\"},{\"name\":\"mail_subject\",\"type\":\"string\"},{\"name\":\"mail_subject_charset\",\"type\":\"string\"},{\"name\":\"mail_content\",\"type\":\"string\"},{\"name\":\"mail_content_charset\",\"type\":\"string\"},{\"name\":\"mail_attachment_name\",\"type\":\"string\"},{\"name\":\"mail_attachment_name_charset\",\"type\":\"string\"},{\"name\":\"mail_attachment_content\",\"type\":\"string\"},{\"name\":\"mail_eml_file\",\"type\":\"string\"},{\"name\":\"mail_snapshot\",\"type\":\"string\"},{\"name\":\"dns_message_id\",\"type\":\"long\"},{\"name\":\"dns_qr\",\"type\":\"long\"},{\"name\":\"dns_opcode\",\"type\":\"long\"},{\"name\":\"dns_aa\",\"type\":\"long\"},{\"name\":\"dns_tc\",\"type\":\"long\"},{\"name\":\"dns_rd\",\"type\":\"long\"},{\"name\":\"dns_ra\",\"type\":\"long\"},{\"name\":\"dns_rcode\",\"type\":\"long\"},{\"name\":\"dns_qdcount\",\"type\":\"long\"},{\"name\":\"dns_ancount\",\"type\":\"long\"},{\"name\":\"dns_nscount\",\"type\":\"long\"},{\"name\":\"dns_arcount\",\"type\":\"long\"},{\"name\":\"dns_qname\",\"type\":\"string\"},{\"name\":\"dns_qtype\",\"type\":\"long\"},{\"name\":\"dns_qclass\",\"type\":\"long\"},{\"name\":\"dns_cname\",\"type\":\"string\"},{\"name\":\"dns_sub\",\"type\":\"long\"},{\"name\":\"dns_rr\",\"type\":\"string\"},{\"name\":\"ssl_version\",\"type\":\"string\"},{\"name\":\"ssl_sni\",\"type\":\"string\"},{\"name\":\"ssl_san\",\"type\":\"string\"},{\"name\":\"ssl_cn\",\"type\":\"string\"},{\"name\":\"ssl_pinningst\",\"type\":\"long\"},{\"name\":\"ssl_intercept_state\",\"type\":\"long\"},{\"name\":\"ssl_passthrough_reason\",\"type\":\"string\"},{\"name\":\"ssl_server_side_latency\",\"type\":\"long\"},{\"name\":\"ssl_client_side_latency\",\"type\":\"long\"},{\"name\":\"ssl_server_side_version\",\"type\":\"string\"},{\"name\":\"ssl_client_side_version\",\"type\":\"string\"},{\"name\":\"ssl_cert_verify\",\"type\":\"long\"},{\"name\":\"ssl_error\",\"type\":\"string\"},{\"name\":\"ssl_con_latency_ms\",\"type\":\"long\"},{\"name\":\"ssl_ja3_fingerprint\",\"type\":\"string\"},{\"name\":\"ssl_ja3_hash\",\"type\":\"string\"},{\"name\":\"ssl_cert_issuer\",\"type\":\"string\"},{\"name\":\"ssl_cert_subject\",\"type\":\"string\"},{\"name\":\"quic_version\",\"type\":\"string\"},{\"name\":\"quic_sni\",\"type\":\"string\"},{\"name\":\"quic_user_agent\",\"type\":\"string\"},{\"name\":\"ftp_account\",\"type\":\"string\"},{\"name\":\"ftp_url\",\"type\":\"string\"},{\"name\":\"ftp_content\",\"type\":\"string\"},{\"name\":\"ftp_link_type\",\"type\":\"string\"},{\"name\":\"bgp_type\",\"type\":\"long\"},{\"name\":\"bgp_as_num\",\"type\":\"string\"},{\"name\":\"bgp_route\",\"type\":\"string\"},{\"name\":\"voip_calling_account\",\"type\":\"string\"},{\"name\":\"voip_called_account\",\"type\":\"string\"},{\"name\":\"voip_calling_number\",\"type\":\"string\"},{\"name\":\"voip_called_number\",\"type\":\"string\"},{\"name\":\"sip_call_id\",\"type\":\"string\"},{\"name\":\"sip_originator_description\",\"type\":\"string\"},{\"name\":\"sip_responder_description\",\"type\":\"string\"},{\"name\":\"sip_user_agent\",\"type\":\"string\"},{\"name\":\"sip_server\",\"type\":\"string\"},{\"name\":\"sip_originator_sdp_connect_ip\",\"type\":\"string\"},{\"name\":\"sip_originator_sdp_media_port\",\"type\":\"long\"},{\"name\":\"sip_originator_sdp_media_type\",\"type\":\"string\"},{\"name\":\"sip_originator_sdp_content\",\"type\":\"string\"},{\"name\":\"sip_responder_sdp_connect_ip\",\"type\":\"string\"},{\"name\":\"sip_responder_sdp_media_port\",\"type\":\"long\"},{\"name\":\"sip_responder_sdp_media_type\",\"type\":\"string\"},{\"name\":\"sip_responder_sdp_content\",\"type\":\"string\"},{\"name\":\"sip_duration_s\",\"type\":\"long\"},{\"name\":\"sip_bye\",\"type\":\"string\"},{\"name\":\"rtp_payload_type_c2s\",\"type\":\"long\"},{\"name\":\"rtp_payload_type_s2c\",\"type\":\"long\"},{\"name\":\"rtp_pcap_path\",\"type\":\"string\"},{\"name\":\"rtp_originator_dir\",\"type\":\"long\"},{\"name\":\"ssh_version\",\"type\":\"string\"},{\"name\":\"ssh_auth_success\",\"type\":\"string\"},{\"name\":\"ssh_client_version\",\"type\":\"string\"},{\"name\":\"ssh_server_version\",\"type\":\"string\"},{\"name\":\"ssh_cipher_alg\",\"type\":\"string\"},{\"name\":\"ssh_mac_alg\",\"type\":\"string\"},{\"name\":\"ssh_compression_alg\",\"type\":\"string\"},{\"name\":\"ssh_kex_alg\",\"type\":\"string\"},{\"name\":\"ssh_host_key_alg\",\"type\":\"string\"},{\"name\":\"ssh_host_key\",\"type\":\"string\"},{\"name\":\"ssh_hassh\",\"type\":\"string\"},{\"name\":\"stratum_cryptocurrency\",\"type\":\"string\"},{\"name\":\"stratum_mining_pools\",\"type\":\"string\"},{\"name\":\"stratum_mining_program\",\"type\":\"string\"},{\"name\":\"streaming_media_url\",\"type\":\"string\"},{\"name\":\"streaming_media_protocol\",\"type\":\"string\"},{\"name\":\"app_extra_info\",\"type\":\"string\"}]}"; + Object parse = JSONObject.parse(json); + + System.err.println(parse); + } +} +*/ |