diff options
Diffstat (limited to 'src/main/java/com/mesasoft/cn/sketch/api')
| -rw-r--r-- | src/main/java/com/mesasoft/cn/sketch/api/BrightCloud.java | 193 | ||||
| -rw-r--r-- | src/main/java/com/mesasoft/cn/sketch/api/ChinaZ.java | 336 |
2 files changed, 529 insertions, 0 deletions
diff --git a/src/main/java/com/mesasoft/cn/sketch/api/BrightCloud.java b/src/main/java/com/mesasoft/cn/sketch/api/BrightCloud.java new file mode 100644 index 0000000..3a51220 --- /dev/null +++ b/src/main/java/com/mesasoft/cn/sketch/api/BrightCloud.java @@ -0,0 +1,193 @@ +package com.mesasoft.cn.sketch.api; + +import com.alibaba.fastjson.JSON; +import com.alibaba.fastjson.JSONArray; +import com.alibaba.fastjson.JSONObject; +import com.mesasoft.cn.sketch.config.ApplicationConfig; +import com.mesasoft.cn.sketch.entity.DomainCategory; +import com.mesasoft.cn.util.FileUtils; +import com.mesasoft.cn.util.ValidationUtils; +import org.apache.log4j.Logger; + +import java.io.*; +import java.net.HttpURLConnection; +import java.net.URL; +import java.nio.charset.StandardCharsets; +import java.util.*; + +/** + * @author yjy + * @version 1.0 + * @date 2021/2/22 2:37 下午 + */ + +public class BrightCloud { + private static final Logger LOG = Logger.getLogger(BrightCloud.class); + + private final Integer maxObjectNum = ApplicationConfig.API_BC_MAXIMUM_QUERYNUM; + private final HashMap<Integer, List<String>> catId2Info = new HashMap<>(); + private HttpURLConnection con; + + public List<DomainCategory> getQueryFiles(List<String> domains){ + JSONObject queryResults = getQueryResults(domains); + return responseSparse(queryResults); + } + + // 获取json格式查询结果 + public JSONObject getQueryResults(List<String> domains) { + if (domains.size()> ApplicationConfig.API_BC_MAXIMUM_QUERYNUM){ + LOG.warn("Too many domains in a http post request, the number of fqdn/url should be no more than " + + ApplicationConfig.API_BC_MAXIMUM_QUERYNUM + "!"); + } + JSONObject jsonRes = new JSONObject(); + try { + URL url = new URL(ApplicationConfig.API_BC_URL); + con = (HttpURLConnection) url.openConnection(); + con.setRequestMethod(ApplicationConfig.API_BC_METHOD); + con.setDoOutput(true); + con.setDoInput(true); + + con.setRequestProperty("Content-Type", "application/json"); + + JSONObject param = new JSONObject(); + param.put("oemid", ApplicationConfig.API_BC_OEMID); + param.put("deviceid", ApplicationConfig.API_BC_DEVICEID); + param.put("uid", ApplicationConfig.API_BC_UID); + + param.put("queries", Collections.singletonList(ApplicationConfig.API_BC_QUERYTYPE)); + param.put("a1cat", ApplicationConfig.API_BC_ISA1CAT); + param.put("reputation", ApplicationConfig.API_BC_ISREPU); + param.put("xml", ApplicationConfig.API_BC_ISXML); // json or xml格式 + + param.put("urls", domains); + + //建立实际的连接 + con.connect(); + OutputStreamWriter writer = new OutputStreamWriter(this.con.getOutputStream(), StandardCharsets.UTF_8); + writer.write(param.toString()); + writer.flush(); + } catch (IOException e) { + e.printStackTrace(); + } + + try { + // 获取服务端响应，通过输入流来读取URL的响应 + InputStream is = con.getInputStream(); + BufferedReader reader = new BufferedReader(new InputStreamReader(is, StandardCharsets.UTF_8)); + StringBuilder sbf = new StringBuilder(); + String strRead = null; + while ((strRead = reader.readLine()) != null) { + sbf.append(strRead); + sbf.append("\r\n"); + } + reader.close(); + + jsonRes = JSONObject.parseObject(sbf.toString()); + con.disconnect(); + } catch (IOException e) { + e.printStackTrace(); + } + return jsonRes; + } + + // json响应内容解析 + public List<DomainCategory> responseSparse(JSONObject records){ + List<DomainCategory> domainFiles = new ArrayList<>(); + Boolean querySucess = records.get("status").equals(200); + + if (!querySucess) { + System.out.print(records); + LOG.error("Wrong query. Query type: " + records.get("type")); + } else { + JSONArray array = records.getJSONArray("results"); + for (int i = 0; i < array.size(); i++) { + JSONObject jo = array.getJSONObject(i); + + // json处理 + JSONObject queries = jo.getJSONObject("queries"); + JSONObject getInfo = queries.getJSONObject(ApplicationConfig.API_BC_QUERYTYPE); + + JSONObject cat = getInfo.getJSONArray("cats").getJSONObject(0); + Integer catId = cat.getInteger("catid"); + String fqdn = jo.getString("url"); + domainFiles.add(new DomainCategory( + fqdn, + "brightcloud", + querySucess, + ValidationUtils.getMatchPattern(fqdn), + getInfo.getInteger("reputation"), + getRepLevel(getInfo.getInteger("reputation")), + catId, + getCatInfo(catId).get(0), + getCatInfo(catId).get(1), + cat.getInteger("conf"), + getInfo.getBoolean("a1cat"))); + } + } + return domainFiles; + } + + private String getRepLevel(Integer repScore){ + String level = null; //用str存放数据 + if (repScore > 80){ level="Trustworthy";} + else if (repScore > 60){ level="Low Risk";} + else if (repScore > 40){ level="Moderate Risk";} + else if (repScore > 20){ level="Suspicious";} + else if (repScore > 0){ level="High Risk";} + return level; + } + + // 获取类别id对应信息 + public void geneCatInfo(){ + if (catId2Info.size()==0){ + JSONObject jsonObject; +// String filePath = Objects.requireNonNull(BrightCloud.class.getClassLoader() +// .getResource(ApplicationConfig.API_BC_CATEINFO_FILE)).getFile(); + String filePath =ApplicationConfig.API_BC_CATEINFO_FILE; + String s = FileUtils.readJsonFile(filePath); + jsonObject = JSON.parseObject(s); + + if (!(jsonObject==null)){ + JSONObject tmp = (JSONObject) jsonObject.getJSONArray("results").get(0); + JSONArray catInfoArray = tmp.getJSONObject("queries").getJSONObject("getcatlist").getJSONArray("cats"); + + for (int i = 0; i < catInfoArray.size(); i++){ + JSONObject keyObject = catInfoArray.getJSONObject(i); + List<String> value = new ArrayList<>(Arrays.asList( + keyObject.getString("catname"), + keyObject.getString("catgroup"))); + catId2Info.put(i+1, value); + } + } + } + } + + public List<String> getCatInfo(Integer catId){ + List<String> info = Arrays.asList("", ""); + + if (0 < catId && catId <= 83) { + if (catId2Info.size()==0){ + geneCatInfo(); + } + + info = catId2Info.get(catId); + + if (info == null){ + LOG.error("Failed at geneCatInfo function"); + System.out.print("Failed at geneCatInfo function"); + } + } + + return info; + } + + public Integer getMaxObjectNum() { + return maxObjectNum; + } + + public static void main(String[] args) { + JSONObject queryResults = new BrightCloud().getQueryResults(Arrays.asList("baidu.com")); + new BrightCloud().responseSparse(queryResults); + } +} + diff --git a/src/main/java/com/mesasoft/cn/sketch/api/ChinaZ.java b/src/main/java/com/mesasoft/cn/sketch/api/ChinaZ.java new file mode 100644 index 0000000..1c75093 --- /dev/null +++ b/src/main/java/com/mesasoft/cn/sketch/api/ChinaZ.java @@ -0,0 +1,336 @@ +package com.mesasoft.cn.sketch.api; +/* + * @Description: + * @Author: chenxu + * @Date: 2021-12-27 13:59:29 + * @LastEditTime: 2021-12-29 17:05:45 + * @LastEditors: chenxu + */ + +import com.alibaba.fastjson.JSONArray; +import com.alibaba.fastjson.JSONObject; +import com.mesasoft.cn.sketch.config.ApplicationConfig; +import com.mesasoft.cn.sketch.entity.DomainWhois; +import org.apache.http.HttpResponse; +import org.apache.http.NameValuePair; +import org.apache.http.client.entity.UrlEncodedFormEntity; +import org.apache.http.client.methods.HttpPost; +import org.apache.http.impl.client.CloseableHttpClient; +import org.apache.http.impl.client.HttpClients; +import org.apache.http.message.BasicNameValuePair; +import org.apache.log4j.Logger; + +import java.io.BufferedReader; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.net.URI; +import java.nio.charset.StandardCharsets; +import java.sql.Date; +import java.util.*; +import java.util.concurrent.TimeUnit; + +public class ChinaZ { + private static final Logger LOG = Logger.getLogger(ChinaZ.class); + private final String apiKey = ApplicationConfig.API_CHINAZ_KEY; + + public List<DomainWhois> getQueryFiles(List<String> objectList) { + List<JSONObject> queryResults = getQueryResults(objectList); + return responseSparse(queryResults); + } + + /** + * @Description: 站长之家单查询 + * @Param : 域名 + * @Return: 查询结果 + */ + public JSONObject getQueryResult(String domain){ + String urlString = ApplicationConfig.API_CHINAZ_URL_SINGLE; + Map<String, String> params = new LinkedHashMap<String,String>(); + params.put("key", apiKey); + params.put("domain", domain); + return whoisInfoResolve(doPost(urlString, params),domain); + } + + /** + * @Description: 站长之家多域名查询 + * @Param : 域名 + * @Return: 查询结果 + */ + public List<JSONObject> getQueryResults(List<String> domainList){ + String urlString = ApplicationConfig.API_CHINAZ_URL_SINGLE; + List<JSONObject> whoisInfoList = new ArrayList<>(); + for (String s : domainList) { + Map<String, String> params = new LinkedHashMap<String, String>(); + params.put("key", apiKey); + params.put("domain", s); + JSONObject r = doPost(urlString, params); + whoisInfoList.add(whoisInfoResolve(r, s)); + } + return whoisInfoList; + } + + public List<DomainWhois> responseSparse(List<JSONObject> records){ + List<DomainWhois> whoisFiles = new ArrayList<>(); + + for(JSONObject record: records) { + Boolean querySucess = record.getBoolean("isSuccess"); + if (!querySucess) { + LOG.error("Failed query. Query response: " + record); + break; + } + + String fqdn = record.getString("domain_name"); + String domainName = record.getString("domain_host"); + Integer matchPattern = fqdn.equals(domainName)? 1 : 2 ; + String source = "chinaz"; + + // json处理 + Date creatDate = null; + Date expiraDate = null; + java.util.Date tmpDate = record.getDate("domain_whois_create_time"); + if(tmpDate!=null){ + creatDate = new Date(tmpDate.getTime()); + } + tmpDate = record.getDate("domain_whois_expiration_time"); + if(tmpDate!=null){ + expiraDate = new Date(tmpDate.getTime()); + } + whoisFiles.add(new DomainWhois( + fqdn, + source, + matchPattern, + record.getBoolean("isSuccess"), + record.getString("domain_host"), + null, + creatDate, + expiraDate, + record.getString("domain_whois_email"), + record.getString("domain_whois_name_servers"), + record.getString("domain_whois_registrar"), + null, + null, + null, + null, + null, + null, + null, + record.getString("domain_whois_phone") + )); + } + return whoisFiles; + } + /** + * @Description: 解析并重构JSON串 + * @Param : 查询得到的“单个”JSON串 + * @Return: 返回重构的JSON串 + */ + public JSONObject whoisInfoResolve(JSONObject jsonRes,String queryDomain){ + JSONObject whoisInfo = new JSONObject(true); + JSONObject res = jsonRes.getJSONObject("Result"); + if(jsonRes.get("StateCode").equals(1)){ + whoisInfo.put("isSuccess", jsonRes.get("StateCode")); + whoisInfo.put("domain_name",queryDomain); + whoisInfo.put("domain_host", res.get("Host")); + whoisInfo.put("domain_whois_create_time", res.get("CreationDate")); + whoisInfo.put("domain_whois_expiration_time", res.get("ExpirationDate")); + whoisInfo.put("domain_whois_registrar", res.get("Registrar")); + whoisInfo.put("whois_registrar_name", res.get("ContactPerson")); + whoisInfo.put("domain_whois_email", res.get("Email")); + whoisInfo.put("domain_whois_phone", res.get("Phone")); + whoisInfo.put("domain_whois_name_servers", res.get("DnsServer")); + whoisInfo.put("domain_whois_status", res.get("DomainStatus")); + }else{ + whoisInfo.put("isSuccess", jsonRes.get("StateCode")); + } + return whoisInfo; + } + + /** + * @Description: 构造批量查询需要的URL + * @Param : 待查询域名 + * @Return: 拼接好的URL + */ + public List<String> queryStringBuilder(List<String> domainList){ + //将域名每50个划分一组 + int CHINAZ_REQUEST_LIMIT = 50 ; + int domainListSize = domainList.size(); + int toIndex = CHINAZ_REQUEST_LIMIT; + Map domainListMap = new HashMap(); + int keyToken = 0; + for(int i = 0;i<domainList.size();i+=CHINAZ_REQUEST_LIMIT){ + if(i+CHINAZ_REQUEST_LIMIT>domainListSize){ //作用为toIndex最后没有50条数据则剩余几条newList中就装几条 + toIndex=domainListSize-i; + } + List<String> newList = domainList.subList(i,i+toIndex); + domainListMap.put("keyName"+keyToken, newList); + keyToken++; + } + //将批量查询的域名，构造成CHINAZ的格式 + List<String> domainListString = new ArrayList<>(); + Iterator iter = domainListMap.entrySet().iterator(); + while (iter.hasNext()) { + Map.Entry entry = (Map.Entry) iter.next(); + Object key = entry.getKey(); + Object val = entry.getValue(); + String urlString = ""; + urlString = String.valueOf(val); + urlString = urlString.replace(", ","|"); + urlString =urlString.replace("[","").replace("]",""); + domainListString.add(urlString); + } + return domainListString; + } + + /** + * @Description: 站长之家批量查询-批量提交查询请求，并获得提取任务ID + * @Param : 域名集合（不能超过50个） + * @Return: 查询结果 + */ + public String batchRequest_step1(String domainsString){ + String TaskID = ""; + String urlString = ApplicationConfig.API_CHINAZ_URL_BATCH; + + Map<String, String> params = new LinkedHashMap<String,String>(); + if (!Objects.equals(domainsString, "overflow")){ + params.put("domains",domainsString); + params.put("key", apiKey); + JSONObject r = doPost(urlString, params); + TaskID = r.get("TaskID").toString(); + return TaskID; + }else{ + return TaskID; + } + + } + + /** + * @Description: 站长之家查询-根据提取任务ID查询数据是否采集完成，如果完成则得到Json格式结果 + * @Param : 任务ID + * @Return: 查询结果 + */ + public JSONObject batchRequest_step2(String TaskID){ + String urlString = ApplicationConfig.API_CHINAZ_URL_BATCH; + + Map<String, String> params = new LinkedHashMap<String,String>(); + params.put("taskid",TaskID); + JSONObject requestTotal = null; + requestTotal = doPost(urlString, params); + return requestTotal; + } + + /** + * @Description: 完成如下内容：1）将domain拼接成50个一组；2）调用step_1的API上传数据；3）调用step_2的API获取数据；4）格式整理，输出数据 + * @Param : 域名列表 + * @Return: whois记录列表 + */ + public List<String> batchRequestController(List<String> domainList){ + + List<String> result = new ArrayList<>(); + if (domainList.size()> 5000){ + System.out.println("Too many urls in a http post request!"); + } + List<String> domainListString = new ArrayList<>(); + List<String> TaskID = new ArrayList<>(); + + + // Queue<String> queue = new LinkedList<String>(); + domainListString = queryStringBuilder(domainList); + //循环发送请求，收集每个请求的TaskID + for (String domainParam : domainListString) { + TaskID.add(batchRequest_step1(domainParam)); + } + + for (String s : TaskID) { + int flag = 0; + //查询接口数据，如果API仍在查询中，则等待10秒继续访问 + while (flag == 0) { + JSONObject data = batchRequest_step2(s); + if (data.get("StateCode").equals(0)) { + long timeToSleep = 10L; + TimeUnit time = TimeUnit.SECONDS; + try { + time.sleep(timeToSleep); + } catch (InterruptedException e) { + System.out.println("Interrupted " + "while Sleeping"); + } + } else { + flag = 1; + + JSONObject json_result = data.getJSONObject("Result"); + + JSONArray json_data = json_result.getJSONArray("Data"); + // //对Data内部的数据进行遍历 + for (int j = 0; j < json_data.size(); j++) { + String queryDomain = (String) json_data.getJSONObject(j).get("Domain"); + result.add(whoisInfoResolve(json_data.getJSONObject(j), queryDomain).toJSONString()); + } + } + } + } + + return result; + } + + /** + * @Description: POST调用API数据 + * @Param : 请求对URL，POST请求体需要添加的 k-v 数据 + * @Return: API JSON数据 + */ + public JSONObject doPost(String url, Map params){ + JSONObject jsonRes = null; + try { + // 定义HttpClient + CloseableHttpClient client = HttpClients.createDefault(); + // 实例化HTTP方法 + HttpPost request = new HttpPost(); + request.setURI(new URI(url)); + + //设置参数 + List<NameValuePair> nvps = new ArrayList<NameValuePair>(); + for (Object o : params.keySet()) { + String name = (String) o; + String value = String.valueOf(params.get(name)); + nvps.add(new BasicNameValuePair(name, value)); + + //System.out.println(name +"-"+value); + } + request.setEntity(new UrlEncodedFormEntity(nvps)); + //发送请求 + HttpResponse httpResponse = client.execute(request); + // 获取响应输入流 + InputStream inStream = httpResponse.getEntity().getContent(); + //对放回数据进行处理 + BufferedReader reader = new BufferedReader(new InputStreamReader(inStream , StandardCharsets.UTF_8)); + StringBuilder strber = new StringBuilder(); + StringBuilder sbf = new StringBuilder(); + String strRead = null; + while ((strRead = reader.readLine()) != null) { + sbf.append(strRead); + sbf.append("\r\n"); + } + // 关闭输入流 + inStream.close(); + jsonRes = JSONObject.parseObject(sbf.toString()); + }catch (Exception e) { + System.out.println("请求接口异常"); + } + return jsonRes; + } + + + public static void main(String[] args){ + ChinaZ t = new ChinaZ(); + + //单查询测试 +// System.out.println(t.singleRequest("aaa.baidu.com")); + + //批量查询测试 + List<String> domainList = new ArrayList<>(); + domainList.add("www.baidu.com"); +// domainList.add("aaa.qq.com"); +// domainList.add("doc.mesalab.com"); + +// System.out.println(t.batchRequestController(domainList)); + System.out.println(t.getQueryResults(domainList)); + } +} |