Merge branch 'develop' of https://git.mesalab.cn/galaxy/galaxy-service.git into develop

6 files changed, 346 insertions, 6 deletions

diff --git a/src/main/java/com/nis/domain/restful/NtcMailLog.java b/src/main/java/com/nis/domain/restful/NtcMailLog.java
index a6ed73c..09782f3 100644
--- a/src/main/java/com/nis/domain/restful/NtcMailLog.java
+++ b/src/main/java/com/nis/domain/restful/NtcMailLog.java
@@ -27,8 +27,19 @@ public class NtcMailLog extends LogEntity{
     protected String searchMailFrom;	//发件人

     protected String searchMailTo; 	//收件人

     protected String searchSubject;		//主题

+    

+    protected String flag;//泛收的标志

+

+	@JsonIgnore

+    public String getFlag() {

+		return flag;

+	}

+

+	public void setFlag(String flag) {

+		this.flag = flag;

+	}

 

-    public String getMailProto() {

+	public String getMailProto() {

         return mailProto;

     }

 

diff --git a/src/main/java/com/nis/domain/restful/NtcSslRecordLog.java b/src/main/java/com/nis/domain/restful/NtcSslRecordLog.java
new file mode 100644
index 0000000..c889104
--- /dev/null
+++ b/src/main/java/com/nis/domain/restful/NtcSslRecordLog.java
@@ -0,0 +1,77 @@
+package com.nis.domain.restful;
+
+import com.fasterxml.jackson.annotation.JsonIgnore;
+import com.nis.domain.LogEntity;
+import com.wordnik.swagger.annotations.ApiModelProperty;
+
+/**
+ * Created by darnell on 2018/6/11.
+ */
+public class NtcSslRecordLog extends LogEntity {
+
+    private static final long serialVersionUID = 164434539180653065L;
+
+
+    @ApiModelProperty(value="版本信息", required=true)
+    protected String version;
+
+    @ApiModelProperty(value="Server Name Indication", required=true, notes = "主要解决一台服务器只能使用一个证书-域名的缺点，通过sni可支持一台服务器为多台域名提供服务")
+    protected String sni;
+
+    @ApiModelProperty(value="SAN-Subject Alternative Name Certificate", required=true, notes = "数字证书的一种形式，运行SSL证书绑定多个域名和子域名。另EV证书，扩展验证证书，只能绑定一个域名")
+    protected String san;
+
+    @ApiModelProperty(value="证书授权机构", required=true)
+    protected String cn;
+
+    @ApiModelProperty(value="SNI", required=true)
+    protected String searchSni ;
+    
+	
+    public String getVersion() {
+        return version;
+    }
+
+    public void setVersion(String version) {
+        this.version = version;
+    }
+
+    public String getSni() {
+        return sni;
+    }
+
+    public void setSni(String sni) {
+        this.sni = sni;
+    }
+
+    public String getSan() {
+        return san;
+    }
+
+    public void setSan(String san) {
+        this.san = san;
+    }
+
+    public String getCn() {
+        return cn;
+    }
+
+    public void setCn(String cn) {
+        this.cn = cn;
+    }
+    
+    /**
+	 * @param searchSni the searchSni to set
+	 */
+	public void setSearchSni(String searchSni) {
+		this.searchSni = searchSni;
+	}
+	
+	/**
+	 * @return the searchSni
+	 */
+	@JsonIgnore
+	public String getSearchSni() {
+		return searchSni;
+	}
+}
diff --git a/src/main/java/com/nis/web/controller/restful/NtcLogSearchController.java b/src/main/java/com/nis/web/controller/restful/NtcLogSearchController.java
index 9ac09b3..00af934 100644
--- a/src/main/java/com/nis/web/controller/restful/NtcLogSearchController.java
+++ b/src/main/java/com/nis/web/controller/restful/NtcLogSearchController.java
@@ -34,6 +34,7 @@ import com.nis.domain.restful.NtcP2pLog;
 import com.nis.domain.restful.NtcPptpLog;
 import com.nis.domain.restful.NtcSshLog;
 import com.nis.domain.restful.NtcSslLog;
+import com.nis.domain.restful.NtcSslRecordLog;
 import com.nis.domain.restful.NtcStreamingMediaLog;
 import com.nis.domain.restful.NtcVoipLog;
 import com.nis.domain.restful.dashboard.TrafficAsnStatistic;
@@ -159,7 +160,8 @@ public class NtcLogSearchController extends BaseRestController {
 
 		return serviceLogResponse(auditLogThread, System.currentTimeMillis() - start, request, "Dns日志检索成功", page, 0);
 	}
-
+	
+	//日志  Mail  和  Mail泛收 用同一个接口
 	@RequestMapping(value = "/ntcMailLogs", method = RequestMethod.GET)
 	@ApiOperation(value = "EMAIL日志查询", httpMethod = "GET", notes = "对应配置为“邮件管理”，存储动作为阻断与监测的命中日志。对日志功能“邮件管理”提供数据基础查询服务。")
 	public Map<String, ?> ntcMailLogs(Page page, NtcMailLog ntcMailLog, Model model, HttpServletRequest request,
@@ -167,6 +169,10 @@ public class NtcLogSearchController extends BaseRestController {
 		long start = System.currentTimeMillis();
 		AuditLogThread auditLogThread = super.saveRequestLog(servicesRequestLogService, Constants.OPACTION_GET, request,
 				null);
+		//判断searchCfgId是否为null
+		if(ntcMailLog.getSearchCfgId() ==null){
+			ntcMailLog.setSearchCfgId("-1");
+		}
 		try {
 			resetTime(ntcMailLog);
 			ntcLogService.queryConditionCheck(auditLogThread, start, ntcMailLog, NtcMailLog.class, page);
@@ -864,4 +870,34 @@ public class NtcLogSearchController extends BaseRestController {
 		return serviceLogResponse(auditLogThread, System.currentTimeMillis() - start, request, "ASN通联关系(源,目的)检索成功",
 				page, 0);
 	}
+	
+	@RequestMapping(value = "/ntcSslRecordLogs", method = RequestMethod.GET)
+	@ApiOperation(value = "SSL泛收日志查询", httpMethod = "GET", notes = "对应配置为“网站管理-SSL”，存储动作为阻断与监测的命中日志。对日志功能“网站管理-SSL”提供数据基础查询服务。")
+	public Map<String, ?> ntcSslRecordLogs(Page page, NtcSslRecordLog ntcSslRecordLog, Model model, HttpServletRequest request,
+			HttpServletResponse response) {
+		long start = System.currentTimeMillis();
+		AuditLogThread auditLogThread = super.saveRequestLog(servicesRequestLogService, Constants.OPACTION_GET, request,
+				null);
+
+		try {
+			resetTime(ntcSslRecordLog);
+			ntcLogService.queryConditionCheck(auditLogThread, start, ntcSslRecordLog, NtcSslRecordLog.class, page);
+			logDataService.getData(page, ntcSslRecordLog);
+		} catch (Exception e) {
+			auditLogThread.setExceptionInfo("SSL泛收日志检索失败:" + e.getMessage());
+			logger.error("SSL泛收日志检索失败：" + ExceptionUtil.getExceptionMsg(e));
+			if (e instanceof RestServiceException) {
+				throw new RestServiceException(auditLogThread, System.currentTimeMillis() - start,
+						"SSL泛收日志检索失败：" + e.getMessage(), ((RestServiceException) e).getErrorCode());
+			} else if (e instanceof ServiceRuntimeException) {
+				throw new ServiceRuntimeException(auditLogThread, System.currentTimeMillis() - start,
+						"SSL泛收日志检索失败：" + e.getMessage(), ((ServiceRuntimeException) e).getErrorCode());
+			} else {
+				throw new ServiceRuntimeException(auditLogThread, System.currentTimeMillis() - start,
+						"SSL泛收日志检索失败：" + e.getMessage(), RestBusinessCode.service_runtime_error.getValue());
+			}
+		}
+
+		return serviceLogResponse(auditLogThread, System.currentTimeMillis() - start, request, "SSL泛收日志检索成功", page, 0);
+	}
 }
diff --git a/src/main/java/com/nis/web/dao/DfLogSearchDao.xml b/src/main/java/com/nis/web/dao/DfLogSearchDao.xml
index 2782054..ac25120 100644
--- a/src/main/java/com/nis/web/dao/DfLogSearchDao.xml
+++ b/src/main/java/com/nis/web/dao/DfLogSearchDao.xml
@@ -1534,4 +1534,39 @@
 		

 		</resultMap>

 	

+	<resultMap id="NtcSslRecordLogMap" type="com.nis.domain.restful.NtcSslRecordLog">

+		<result column="cfg_id" jdbcType="BIGINT" property="cfgId" />

+		<result column="found_Time" jdbcType="TIMESTAMP" property="foundTime" />

+		<result column="recv_Time" jdbcType="TIMESTAMP" property="recvTime" />

+		<result column="trans_proto" jdbcType="VARCHAR" property="transProto" />

+		<result column="addr_type" jdbcType="INTEGER" property="addrType" />

+		<result column="d_ip" jdbcType="VARCHAR" property="dIp" />

+		<result column="s_ip" jdbcType="VARCHAR" property="sIp" />

+		<result column="d_port" jdbcType="VARCHAR" property="dPort" />

+		<result column="s_port" jdbcType="VARCHAR" property="sPort" />

+		<result column="service" jdbcType="INTEGER" property="service" />

+		<result column="entrance_id" jdbcType="BIGINT" property="entranceId" />

+		<result column="device_id" jdbcType="INTEGER" property="deviceId" />

+		<result column="direction" jdbcType="INTEGER" property="direction" />

+		<result column="stream_dir" jdbcType="INTEGER" property="streamDir" />

+		<result column="cap_ip" jdbcType="VARCHAR" property="capIp" />

+		<result column="addr_list" jdbcType="VARCHAR" property="addrList" />

+		<result column="user_region" jdbcType="VARCHAR" property="userRegion" />

+		<result column="server_locate" jdbcType="VARCHAR" property="serverLocate" />

+		<result column="client_locate" jdbcType="VARCHAR" property="clientLocate" />

+		<result column="s_asn" jdbcType="VARCHAR" property="sAsn" />

+		<result column="d_asn" jdbcType="VARCHAR" property="dAsn" />

+		<result column="s_subscribe_id" jdbcType="VARCHAR" property="sSubscribeId" />

+		<result column="d_subscribe_id" jdbcType="VARCHAR" property="dSubscribeId" />

+		<result column="scene_file" jdbcType="VARCHAR" property="sceneFile" />

+		<result column="link_id" jdbcType="BIGINT" property="linkId" />

+		<result column="encap_type" jdbcType="INTEGER" property="encapType" />

+		<result column="inner_smac" jdbcType="VARCHAR" property="innerSmac" />

+		<result column="inner_dmac" jdbcType="VARCHAR" property="innerDmac" />

+

+		<result column="version" jdbcType="VARCHAR" property="version" />

+		<result column="sni" jdbcType="VARCHAR" property="sni" />

+		<result column="san" jdbcType="VARCHAR" property="san" />

+		<result column="cn" jdbcType="VARCHAR" property="cn" />

+	</resultMap>

 </mapper>
\ No newline at end of file
diff --git a/src/main/java/com/nis/web/service/LogDataService.java b/src/main/java/com/nis/web/service/LogDataService.java
index b37b4cb..c51d49d 100644
--- a/src/main/java/com/nis/web/service/LogDataService.java
+++ b/src/main/java/com/nis/web/service/LogDataService.java
@@ -179,7 +179,12 @@ public class LogDataService {
 			orderBy = orderBy + "found_Time desc ";

 		}

 		if (Constants.ISUSECLICKHOUSE) {

-			getDataFromClickHouse(page, obj, tableName, className, orderBy.toLowerCase());

+			//判断实体类中是否存在某个字段

+			if(ifExistFiledName(obj,"flag")){//邮件泛收日志

+				getDataFromClickHouseFS(page, obj, tableName, className, orderBy.toLowerCase());

+			}else{

+				getDataFromClickHouse(page, obj, tableName, className, orderBy.toLowerCase());

+			}

 		} else {

 			getDataFromHive(page, obj, tableName, className, orderBy.toLowerCase());

 		}

@@ -316,7 +321,7 @@ public class LogDataService {
 		}

 		sql.append(orderBy.toLowerCase() + "  limit " + startNum + "," + page.getPageSize());// clickhouse的分页与mysql相同

 

-		if (tableName.toUpperCase().equals("TBS_ODS_NTC_CONN_RECORD_LOG")) {

+		if (tableName.toUpperCase().equals("TBS_ODS_NTC_CONN_RECORD_LOG")|| tableName.toUpperCase().equals("TBS_ODS_NTC_COLLECT_SSL_LOG")) {

 			searchFromLocalCK(page, bean, sql, countSql);

 		} else {

 			searchFromDataCenter(page, bean, sql, countSql);

@@ -635,5 +640,181 @@ public class LogDataService {
 		Method method = bean.getClass().getMethod(methodName, classArr);

 		method.invoke(bean, value);

 	}

+	

+	/**

+	 * 利用反射查询一个对象中是否存在某个属性

+	 * 

+	 * @param obj

+	 * @param fieldName

+	 * @throws Exception

+	 */

+	private static boolean  ifExistFiledName(Object obj,String filedName) throws Exception{

+		Class class1=obj.getClass();//需要检测的类	 

+		/**

+		 * 循环遍历所有的元素，检测有没有这个名字

+		 */

+		Field[] fields=class1.getDeclaredFields();

+		 

+		boolean b=false;

+		for (int i = 0; i < fields.length; i++) {

+		    if(fields[i].getName().equals(filedName))

+		    {

+		        b=true;

+		        break;

+		    }

+		}

+		return b;

+	}

+	

+	/**

+	 * 有泛收的页面调用的方法，(邮箱泛收)

+	 * 从clickhouse中查询数据,注意clickhouse区分大小写,目前和百分点商定都是用小写

+	 * 

+	 * @param page      里面含有pagesize和pageno,order by

+	 * @param bean      日志类对象(从DfLogSearchDao.xml中获取对应的map,类名+Map),用来获取各个属性对应的数据库字段名

+	 * @param tableName 表名

+	 * @param className 类名

+	 * @param orderBy   orderby条件

+	 * @throws Exception

+	 */

+	private <T> void getDataFromClickHouseFS(Page<T> page, Object bean, String tableName, String className,

+			String orderBy) throws Exception {

+		tableName = tableName.toLowerCase();

+		String showColmun = getFiledsSql(className, page.getFields());

+		StringBuffer sql = new StringBuffer();

+		Map<String, String> filedAndColumnMap = getFiledAndColumnMap(bean.getClass());

+		if (null == showColmun || showColmun.equals("")) {

+			for (String key : filedAndColumnMap.keySet()) {

+				if (!filedAndColumnMap.get(key).toLowerCase().equals("id")) {

+					sql.append(filedAndColumnMap.get(key) + ",");

+				}

+			}

+		} else {

+			sql.append(showColmun);

+		}

+		String sqlTrim = sql.toString().trim();

+		if (sqlTrim.endsWith(",")) {

+			sqlTrim = sqlTrim.substring(0, sqlTrim.length() - 1);

+		}

+		sql.setLength(0);

+		sql.append(" select " + sqlTrim.toLowerCase() + "  from " + tableName.toLowerCase() + " t where ");

+		StringBuffer whereFoundTime = new StringBuffer();

+		StringBuffer countSql = new StringBuffer();

+		countSql.append("select count(1) from " + tableName + " where ");

+

+		StringBuffer whereSB = new StringBuffer();

+		if (!StringUtil.isEmpty(bean)) {

+			Class<?> clazz = bean.getClass();

+			Map<String, String> filedsType = null;

+			filedsType = getFiledsType(bean);

+			for (; clazz != Object.class; clazz = clazz.getSuperclass()) {

+				Field[] fields = clazz.getDeclaredFields();

+				for (int i = 0; i < fields.length; i++) {

+					// 现在gwall日志表结构中只有数值和字符串两种类型,数值都是int类型没有bigint所以不需要加L,

+					Field f = fields[i];

+					String key = f.getName();// 获取字段名

+					if (f.getType().getName().equals("java.lang.String") && key.startsWith("search")) {

+						Object value = getFieldValue(bean, key);

+						if (!StringUtil.isEmpty(value)) {

+							setFieldValue(bean, key, value.toString().trim());

+							if (key.endsWith("Time")) {// 日期开始或结束的字段

+								if (col2col.containsKey(key)) {

+									value = sdf.parse(value.toString().trim()).getTime() / 1000;

+									if (col2col.get(key).get("start") != null) {

+										whereFoundTime.append(" and "

+												+ filedAndColumnMap.get(col2col.get(key).get("start")).toLowerCase()

+												+ ">=" + value);

+									} else {

+										whereFoundTime.append(" and "

+												+ filedAndColumnMap.get(col2col.get(key).get("end")).toLowerCase() + "<"

+												+ value);

+									}

+								}

+							} else {

+								if (key.toLowerCase().startsWith("search")) {

+									key = key.replace("search", "");

+									key = key.substring(0, 1).toLowerCase() + key.substring(1);

+								}

+								// clickhouse写法

+								String type = filedsType.get(key).trim();

+								String field = filedAndColumnMap.get(key).toLowerCase();

+								if (type.equals("java.lang.String")) {

+

+									if (field.contains("url") || field.equals("website")) {

+										whereSB.append(" and " + field + " like '"

+												+ StringEscapeUtils.unescapeHtml4(value.toString().trim()) + "%'");

+									} else if (field.equals("client_locate") || field.equals("server_locate")) {

+										whereSB.append(" and " + field + " like '%"

+												+ StringEscapeUtils.unescapeHtml4(value.toString().trim()) + "%'");

+									} else {

+										whereSB.append(" and " + field + "='"

+												+ StringEscapeUtils.unescapeHtml4(value.toString().trim()) + "'");

+									}

+

+								} else if (type.equals("java.lang.Integer") || type.equals("int")

+										|| type.equals("java.lang.Long") || type.equals("long")) {

+										if (field.equals("cfg_id")|| field.equals("web_id") || field.equals("app_id")

+												|| field.equals("proto_id")){

+											if(field.equals("cfg_id")){

+												if(value.toString().trim().equals("0")){

+													whereSB.append(" and " + filedAndColumnMap.get(key).toLowerCase() + " ="

+															+ value.toString().trim());

+												}else if(value.toString().trim().equals("-1")){

+													whereSB.append(" and " + filedAndColumnMap.get(key).toLowerCase() + " >0" );

+												}else{

+													whereSB.append(" and " + filedAndColumnMap.get(key).toLowerCase() + " in("

+															+ value.toString().trim() + ")");

+												}

+											}else{

+													whereSB.append(" and " + filedAndColumnMap.get(key).toLowerCase() + " in("

+															+ value.toString().trim() + ")");

+											}

+										}else {

+											whereSB.append(" and " + filedAndColumnMap.get(key).toLowerCase() + "="

+													+ value.toString().trim());

+										}

+									

+								}

+

+							}

+						}

+

+					}

+				}

 

+			}

+		}

+		Integer startNum = (page.getPageNo() - 1) * page.getPageSize();

+		StringBuffer foundTimeSql = new StringBuffer();

+		foundTimeSql.append("select  found_time  from " + tableName + " where ");

+		Integer limitCount = startNum + page.getPageSize();

+		if (whereSB.length() == 0) {// 没有其他查询条件只有默认的found_time条件

+			if (whereFoundTime.length() > 0) {

+				int indexOf = whereFoundTime.indexOf("and") + "and".length();

+				countSql.append(whereFoundTime.substring(indexOf));

+

+				foundTimeSql

+						.append(whereFoundTime.substring(indexOf) + orderBy.toLowerCase() + "  limit " + limitCount);

+				sql.append(" found_time in(" + foundTimeSql + ") ");

+			} else {

+				throw new RuntimeException("从clickhouse的" + tableName + "表查询时,必须要有一个where条件");

+			}

+		} else {

+			int foundIndexOf = whereFoundTime.append(whereSB).indexOf("and") + "and".length();

+			countSql.append(whereFoundTime.substring(foundIndexOf));

+			foundTimeSql

+					.append(whereFoundTime.substring(foundIndexOf) + orderBy.toLowerCase() + "  limit " + limitCount);

+

+			int indexOf = whereSB.indexOf("and") + "and".length();

+			sql.append(whereSB.substring(indexOf) + " and found_time in(" + foundTimeSql + ") ");

+		}

+		sql.append(orderBy.toLowerCase() + "  limit " + startNum + "," + page.getPageSize());// clickhouse的分页与mysql相同

+

+		if (tableName.toUpperCase().equals("TBS_ODS_NTC_CONN_RECORD_LOG") ) {

+			searchFromLocalCK(page, bean, sql, countSql);

+		} else {

+			searchFromDataCenter(page, bean, sql, countSql);

+		}

+

+	}

 }

diff --git a/src/main/resources/applicationLog-clickhouse.properties b/src/main/resources/applicationLog-clickhouse.properties
index 9845875..6b69dac 100644
--- a/src/main/resources/applicationLog-clickhouse.properties
+++ b/src/main/resources/applicationLog-clickhouse.properties
@@ -1,4 +1,4 @@
-#由于数据中心ClickHouse日志表的表名会变动,所以本系统中将日志的表名提取到配置文件中方便后期修改,key是bean的名称+ClickHouseTable,value是ClickHouseTable的实际表名

+#\u7531\u4e8e\u6570\u636e\u4e2d\u5fc3ClickHouse\u65e5\u5fd7\u8868\u7684\u8868\u540d\u4f1a\u53d8\u52a8,\u6240\u4ee5\u672c\u7cfb\u7edf\u4e2d\u5c06\u65e5\u5fd7\u7684\u8868\u540d\u63d0\u53d6\u5230\u914d\u7f6e\u6587\u4ef6\u4e2d\u65b9\u4fbf\u540e\u671f\u4fee\u6539,key\u662fbean\u7684\u540d\u79f0+ClickHouseTable,value\u662fClickHouseTable\u7684\u5b9e\u9645\u8868\u540d

 NtcIpLogClickHouseTable=TBS_ODS_NTC_IP_LOG

 NtcHttpLogClickHouseTable=TBS_ODS_NTC_HTTP_LOG

 NtcDnsLogClickHouseTable=TBS_ODS_NTC_DNS_LOG

@@ -38,7 +38,7 @@ NtcKeywordsUrlLogClickHouseTable=TBS_ODS_NTC_KEYWORDS_URL_LOG
 NtcCollectVoipLogClickHouseTable=TBS_ODS_NTC_COLLECT_VOIP_LOG

 NtcCollectRadiusLogClickHouseTable=TBS_ODS_NTC_COLLECT_RADIUS_LOG

 NtcConnRecordLogClickHouseTable=TBS_ODS_NTC_CONN_RECORD_LOG

-

+NtcSslRecordLogClickHouseTable=TBS_ODS_NTC_COLLECT_SSL_LOG