summaryrefslogtreecommitdiff
path: root/src/main/java/com/mesasoft/cn/sketch/api/ChinaZ.java
diff options
context:
space:
mode:
Diffstat (limited to 'src/main/java/com/mesasoft/cn/sketch/api/ChinaZ.java')
-rw-r--r--src/main/java/com/mesasoft/cn/sketch/api/ChinaZ.java336
1 files changed, 336 insertions, 0 deletions
diff --git a/src/main/java/com/mesasoft/cn/sketch/api/ChinaZ.java b/src/main/java/com/mesasoft/cn/sketch/api/ChinaZ.java
new file mode 100644
index 0000000..1c75093
--- /dev/null
+++ b/src/main/java/com/mesasoft/cn/sketch/api/ChinaZ.java
@@ -0,0 +1,336 @@
+package com.mesasoft.cn.sketch.api;
+/*
+ * @Description:
+ * @Author: chenxu
+ * @Date: 2021-12-27 13:59:29
+ * @LastEditTime: 2021-12-29 17:05:45
+ * @LastEditors: chenxu
+ */
+
+import com.alibaba.fastjson.JSONArray;
+import com.alibaba.fastjson.JSONObject;
+import com.mesasoft.cn.sketch.config.ApplicationConfig;
+import com.mesasoft.cn.sketch.entity.DomainWhois;
+import org.apache.http.HttpResponse;
+import org.apache.http.NameValuePair;
+import org.apache.http.client.entity.UrlEncodedFormEntity;
+import org.apache.http.client.methods.HttpPost;
+import org.apache.http.impl.client.CloseableHttpClient;
+import org.apache.http.impl.client.HttpClients;
+import org.apache.http.message.BasicNameValuePair;
+import org.apache.log4j.Logger;
+
+import java.io.BufferedReader;
+import java.io.InputStream;
+import java.io.InputStreamReader;
+import java.net.URI;
+import java.nio.charset.StandardCharsets;
+import java.sql.Date;
+import java.util.*;
+import java.util.concurrent.TimeUnit;
+
+public class ChinaZ {
+ private static final Logger LOG = Logger.getLogger(ChinaZ.class);
+ private final String apiKey = ApplicationConfig.API_CHINAZ_KEY;
+
+ public List<DomainWhois> getQueryFiles(List<String> objectList) {
+ List<JSONObject> queryResults = getQueryResults(objectList);
+ return responseSparse(queryResults);
+ }
+
+ /**
+ * @Description: 站长之家单查询
+ * @Param : 域名
+ * @Return: 查询结果
+ */
+ public JSONObject getQueryResult(String domain){
+ String urlString = ApplicationConfig.API_CHINAZ_URL_SINGLE;
+ Map<String, String> params = new LinkedHashMap<String,String>();
+ params.put("key", apiKey);
+ params.put("domain", domain);
+ return whoisInfoResolve(doPost(urlString, params),domain);
+ }
+
+ /**
+ * @Description: 站长之家多域名查询
+ * @Param : 域名
+ * @Return: 查询结果
+ */
+ public List<JSONObject> getQueryResults(List<String> domainList){
+ String urlString = ApplicationConfig.API_CHINAZ_URL_SINGLE;
+ List<JSONObject> whoisInfoList = new ArrayList<>();
+ for (String s : domainList) {
+ Map<String, String> params = new LinkedHashMap<String, String>();
+ params.put("key", apiKey);
+ params.put("domain", s);
+ JSONObject r = doPost(urlString, params);
+ whoisInfoList.add(whoisInfoResolve(r, s));
+ }
+ return whoisInfoList;
+ }
+
+ public List<DomainWhois> responseSparse(List<JSONObject> records){
+ List<DomainWhois> whoisFiles = new ArrayList<>();
+
+ for(JSONObject record: records) {
+ Boolean querySucess = record.getBoolean("isSuccess");
+ if (!querySucess) {
+ LOG.error("Failed query. Query response: " + record);
+ break;
+ }
+
+ String fqdn = record.getString("domain_name");
+ String domainName = record.getString("domain_host");
+ Integer matchPattern = fqdn.equals(domainName)? 1 : 2 ;
+ String source = "chinaz";
+
+ // json处理
+ Date creatDate = null;
+ Date expiraDate = null;
+ java.util.Date tmpDate = record.getDate("domain_whois_create_time");
+ if(tmpDate!=null){
+ creatDate = new Date(tmpDate.getTime());
+ }
+ tmpDate = record.getDate("domain_whois_expiration_time");
+ if(tmpDate!=null){
+ expiraDate = new Date(tmpDate.getTime());
+ }
+ whoisFiles.add(new DomainWhois(
+ fqdn,
+ source,
+ matchPattern,
+ record.getBoolean("isSuccess"),
+ record.getString("domain_host"),
+ null,
+ creatDate,
+ expiraDate,
+ record.getString("domain_whois_email"),
+ record.getString("domain_whois_name_servers"),
+ record.getString("domain_whois_registrar"),
+ null,
+ null,
+ null,
+ null,
+ null,
+ null,
+ null,
+ record.getString("domain_whois_phone")
+ ));
+ }
+ return whoisFiles;
+ }
+ /**
+ * @Description: 解析并重构JSON串
+ * @Param : 查询得到的“单个”JSON串
+ * @Return: 返回重构的JSON串
+ */
+ public JSONObject whoisInfoResolve(JSONObject jsonRes,String queryDomain){
+ JSONObject whoisInfo = new JSONObject(true);
+ JSONObject res = jsonRes.getJSONObject("Result");
+ if(jsonRes.get("StateCode").equals(1)){
+ whoisInfo.put("isSuccess", jsonRes.get("StateCode"));
+ whoisInfo.put("domain_name",queryDomain);
+ whoisInfo.put("domain_host", res.get("Host"));
+ whoisInfo.put("domain_whois_create_time", res.get("CreationDate"));
+ whoisInfo.put("domain_whois_expiration_time", res.get("ExpirationDate"));
+ whoisInfo.put("domain_whois_registrar", res.get("Registrar"));
+ whoisInfo.put("whois_registrar_name", res.get("ContactPerson"));
+ whoisInfo.put("domain_whois_email", res.get("Email"));
+ whoisInfo.put("domain_whois_phone", res.get("Phone"));
+ whoisInfo.put("domain_whois_name_servers", res.get("DnsServer"));
+ whoisInfo.put("domain_whois_status", res.get("DomainStatus"));
+ }else{
+ whoisInfo.put("isSuccess", jsonRes.get("StateCode"));
+ }
+ return whoisInfo;
+ }
+
+ /**
+ * @Description: 构造批量查询需要的URL
+ * @Param : 待查询域名
+ * @Return: 拼接好的URL
+ */
+ public List<String> queryStringBuilder(List<String> domainList){
+ //将域名每50个划分一组
+ int CHINAZ_REQUEST_LIMIT = 50 ;
+ int domainListSize = domainList.size();
+ int toIndex = CHINAZ_REQUEST_LIMIT;
+ Map domainListMap = new HashMap();
+ int keyToken = 0;
+ for(int i = 0;i<domainList.size();i+=CHINAZ_REQUEST_LIMIT){
+ if(i+CHINAZ_REQUEST_LIMIT>domainListSize){ //作用为toIndex最后没有50条数据则剩余几条newList中就装几条
+ toIndex=domainListSize-i;
+ }
+ List<String> newList = domainList.subList(i,i+toIndex);
+ domainListMap.put("keyName"+keyToken, newList);
+ keyToken++;
+ }
+ //将批量查询的域名，构造成CHINAZ的格式
+ List<String> domainListString = new ArrayList<>();
+ Iterator iter = domainListMap.entrySet().iterator();
+ while (iter.hasNext()) {
+ Map.Entry entry = (Map.Entry) iter.next();
+ Object key = entry.getKey();
+ Object val = entry.getValue();
+ String urlString = "";
+ urlString = String.valueOf(val);
+ urlString = urlString.replace(", ","|");
+ urlString =urlString.replace("[","").replace("]","");
+ domainListString.add(urlString);
+ }
+ return domainListString;
+ }
+
+ /**
+ * @Description: 站长之家批量查询-批量提交查询请求，并获得提取任务ID
+ * @Param : 域名集合（不能超过50个）
+ * @Return: 查询结果
+ */
+ public String batchRequest_step1(String domainsString){
+ String TaskID = "";
+ String urlString = ApplicationConfig.API_CHINAZ_URL_BATCH;
+
+ Map<String, String> params = new LinkedHashMap<String,String>();
+ if (!Objects.equals(domainsString, "overflow")){
+ params.put("domains",domainsString);
+ params.put("key", apiKey);
+ JSONObject r = doPost(urlString, params);
+ TaskID = r.get("TaskID").toString();
+ return TaskID;
+ }else{
+ return TaskID;
+ }
+
+ }
+
+ /**
+ * @Description: 站长之家查询-根据提取任务ID查询数据是否采集完成，如果完成则得到Json格式结果
+ * @Param : 任务ID
+ * @Return: 查询结果
+ */
+ public JSONObject batchRequest_step2(String TaskID){
+ String urlString = ApplicationConfig.API_CHINAZ_URL_BATCH;
+
+ Map<String, String> params = new LinkedHashMap<String,String>();
+ params.put("taskid",TaskID);
+ JSONObject requestTotal = null;
+ requestTotal = doPost(urlString, params);
+ return requestTotal;
+ }
+
+ /**
+ * @Description: 完成如下内容：1）将domain拼接成50个一组；2）调用step_1的API上传数据；3）调用step_2的API获取数据；4）格式整理，输出数据
+ * @Param : 域名列表
+ * @Return: whois记录列表
+ */
+ public List<String> batchRequestController(List<String> domainList){
+
+ List<String> result = new ArrayList<>();
+ if (domainList.size()> 5000){
+ System.out.println("Too many urls in a http post request!");
+ }
+ List<String> domainListString = new ArrayList<>();
+ List<String> TaskID = new ArrayList<>();
+
+
+ // Queue<String> queue = new LinkedList<String>();
+ domainListString = queryStringBuilder(domainList);
+ //循环发送请求，收集每个请求的TaskID
+ for (String domainParam : domainListString) {
+ TaskID.add(batchRequest_step1(domainParam));
+ }
+
+ for (String s : TaskID) {
+ int flag = 0;
+ //查询接口数据，如果API仍在查询中，则等待10秒继续访问
+ while (flag == 0) {
+ JSONObject data = batchRequest_step2(s);
+ if (data.get("StateCode").equals(0)) {
+ long timeToSleep = 10L;
+ TimeUnit time = TimeUnit.SECONDS;
+ try {
+ time.sleep(timeToSleep);
+ } catch (InterruptedException e) {
+ System.out.println("Interrupted " + "while Sleeping");
+ }
+ } else {
+ flag = 1;
+
+ JSONObject json_result = data.getJSONObject("Result");
+
+ JSONArray json_data = json_result.getJSONArray("Data");
+ // //对Data内部的数据进行遍历
+ for (int j = 0; j < json_data.size(); j++) {
+ String queryDomain = (String) json_data.getJSONObject(j).get("Domain");
+ result.add(whoisInfoResolve(json_data.getJSONObject(j), queryDomain).toJSONString());
+ }
+ }
+ }
+ }
+
+ return result;
+ }
+
+ /**
+ * @Description: POST调用API数据
+ * @Param : 请求对URL，POST请求体需要添加的 k-v 数据
+ * @Return: API JSON数据
+ */
+ public JSONObject doPost(String url, Map params){
+ JSONObject jsonRes = null;
+ try {
+ // 定义HttpClient
+ CloseableHttpClient client = HttpClients.createDefault();
+ // 实例化HTTP方法
+ HttpPost request = new HttpPost();
+ request.setURI(new URI(url));
+
+ //设置参数
+ List<NameValuePair> nvps = new ArrayList<NameValuePair>();
+ for (Object o : params.keySet()) {
+ String name = (String) o;
+ String value = String.valueOf(params.get(name));
+ nvps.add(new BasicNameValuePair(name, value));
+
+ //System.out.println(name +"-"+value);
+ }
+ request.setEntity(new UrlEncodedFormEntity(nvps));
+ //发送请求
+ HttpResponse httpResponse = client.execute(request);
+ // 获取响应输入流
+ InputStream inStream = httpResponse.getEntity().getContent();
+ //对放回数据进行处理
+ BufferedReader reader = new BufferedReader(new InputStreamReader(inStream , StandardCharsets.UTF_8));
+ StringBuilder strber = new StringBuilder();
+ StringBuilder sbf = new StringBuilder();
+ String strRead = null;
+ while ((strRead = reader.readLine()) != null) {
+ sbf.append(strRead);
+ sbf.append("\r\n");
+ }
+ // 关闭输入流
+ inStream.close();
+ jsonRes = JSONObject.parseObject(sbf.toString());
+ }catch (Exception e) {
+ System.out.println("请求接口异常");
+ }
+ return jsonRes;
+ }
+
+
+ public static void main(String[] args){
+ ChinaZ t = new ChinaZ();
+
+ //单查询测试
+// System.out.println(t.singleRequest("aaa.baidu.com"));
+
+ //批量查询测试
+ List<String> domainList = new ArrayList<>();
+ domainList.add("www.baidu.com");
+// domainList.add("aaa.qq.com");
+// domainList.add("doc.mesalab.com");
+
+// System.out.println(t.batchRequestController(domainList));
+ System.out.println(t.getQueryResults(domainList));
+ }
+}
generated by cgit v1.2.3 (git 2.39.1) at 2025-12-30 15:06:13 +0000