diff options
| author | tanghao <admin@LAPTOP-QCSKVLI9> | 2021-03-16 17:23:36 +0800 |
|---|---|---|
| committer | tanghao <admin@LAPTOP-QCSKVLI9> | 2021-03-16 17:23:36 +0800 |
| commit | 4d72cd078d4ded9b0ac4935a75ab87b1edb05d13 (patch) | |
| tree | 8b5d9a1ede6fd589d029f70cce3066f555f757a6 | |
| parent | 7a020d3235ea7bab29f42e17b4ec2728e2311e1b (diff) | |
feat: 告警静默功能开发
20 files changed, 751 insertions, 23 deletions
diff --git a/nz-admin/src/main/java/com/nis/common/config/AlertMessageSilenceEmailHandle.java b/nz-admin/src/main/java/com/nis/common/config/AlertMessageSilenceEmailHandle.java new file mode 100644 index 00000000..355d9342 --- /dev/null +++ b/nz-admin/src/main/java/com/nis/common/config/AlertMessageSilenceEmailHandle.java @@ -0,0 +1,430 @@ +package com.nis.common.config; + +import com.alibaba.fastjson.JSON; +import com.alibaba.fastjson.JSONObject; +import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper; +import com.baomidou.mybatisplus.core.conditions.query.QueryWrapper; +import com.baomidou.mybatisplus.core.toolkit.ObjectUtils; +import com.nis.common.utils.Constant; +import com.nis.common.utils.DateUtil; +import com.nis.common.utils.MailUtil; +import com.nis.common.utils.ToolUtil; +import com.nis.modules.alert.dao.AlertNotifiScriptDao; +import com.nis.modules.alert.entity.AlertMessageEntity; +import com.nis.modules.alert.entity.AlertNotificationScript; +import com.nis.modules.alert.entity.AlertNotifyLog; +import com.nis.modules.alert.entity.AlertRuleEntity; +import com.nis.modules.alert.service.AlertMessageService; +import com.nis.modules.alert.service.AlertNotifyLogService; +import com.nis.modules.alert.service.AlertRuleService; +import com.nis.modules.alert.service.AlertSilenceService; +import com.nis.modules.sys.dao.SysUserNotificationDao; +import com.nis.modules.sys.entity.SysUserEntity; +import com.nis.modules.sys.entity.SysUserNotification; +import com.nis.modules.sys.service.SysConfigService; +import com.nis.modules.sys.service.SysUserService; + +import cn.hutool.core.util.ObjectUtil; +import cn.hutool.core.util.RuntimeUtil; +import cn.hutool.log.Log; +import freemarker.cache.StringTemplateLoader; +import freemarker.core.ParseException; +import freemarker.template.Configuration; +import freemarker.template.MalformedTemplateNameException; +import freemarker.template.Template; +import freemarker.template.TemplateException; +import freemarker.template.TemplateNotFoundException; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Date; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Map.Entry; +import java.util.Set; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.stream.Collectors; + +import org.apache.commons.collections.CollectionUtils; +import org.apache.commons.lang3.StringUtils; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.CommandLineRunner; +import org.springframework.data.redis.core.ListOperations; +import org.springframework.data.redis.core.RedisTemplate; +import org.springframework.stereotype.Component; +import org.springframework.ui.freemarker.FreeMarkerTemplateUtils; + +@Component +public class AlertMessageSilenceEmailHandle implements CommandLineRunner { + + private static final Log log = Log.get(); + + private static final ExecutorService executorService = Executors.newCachedThreadPool(); + + @Autowired + private RedisTemplate<String, Object> redisTemplate; + + @Autowired + private AlertSilenceService alertSilenceService; + + @Autowired + private AlertMessageService alertMessageService; + + @Autowired + private SysConfigService configService; + + @Autowired + private AlertRuleService alertRuleService; + + @Autowired + private SysUserService sysUserService; + + @Autowired + private SysUserNotificationDao userNotificationDao; + + @Autowired + private AlertNotifiScriptDao alertNotifiScriptDao; + + @Autowired + private AlertNotifyLogService alertNotifyLogService; + + @Autowired + private MailUtil mailUtil; + + private Template template; + + @Override + public void run(String... args) throws Exception { + + executorService.execute(new Runnable() { + + @Override + public void run() { + log.debug("告警静默匹配规则线程运行"); + while(true) { + // redis读取 alert:message:recv数据 + Object obj = redisTemplate.opsForList().rightPop(Constant.ALERT_MESSAGE_RECV); + if(ObjectUtil.isEmpty(obj)) { + //log.debug("缓存中无告警消息入库"); + continue; + } + log.debug("缓存中接收到的告警数据信息为:"+JSON.toJSONString(obj)); + // 告警消息集合 + List<AlertMessageEntity> recvs = null; + try { + recvs = JSONObject.parseArray(obj.toString(), AlertMessageEntity.class); + } catch (Exception e) { + log.error("从redis中取告警消息转化失败,错误信息:" +e + " 转化参数是:" + obj.toString()); + } + + // 最终入库的数据 + List<AlertMessageEntity> entities = new ArrayList<AlertMessageEntity>(); + + // 当前激活状态的告警消息 + List<AlertMessageEntity> pendingAlert = alertMessageService.list(new QueryWrapper<AlertMessageEntity>().lambda().eq(AlertMessageEntity::getState, 1)); + Map<String, AlertMessageEntity> alertHashKeyAndEntityMap = new HashMap<>(); + for (AlertMessageEntity entity : pendingAlert) { + alertHashKeyAndEntityMap.put(entity.getHashKey(), entity); + } + + // 接收告警的当前时间 + Date currentTime = DateUtil.getUTCTimeByConfigTimeZone(); + //获取内置告警规则 + AlertRuleEntity buildRule = alertRuleService.getOne(new QueryWrapper<AlertRuleEntity>().lambda().eq(AlertRuleEntity::getBuildIn, 1)); + Integer buildRuleId = buildRule.getId(); + + // redis读取 alert:message:data数据 + Map<Object, Object> silences = redisTemplate.opsForHash().entries(Constant.ALERT_MESSAGE_SILENCE); + + for(AlertMessageEntity alertRecv : recvs) { + try { + // 匹配静默规则 + boolean checkMatchSilence = alertSilenceService.checkMatchSilence(alertRecv); + log.debug("静默是否匹配 :"+checkMatchSilence +"告警规则id为:"+alertRecv.getRuleId()); + // 匹配上 判断数据是否有缓存 + if(checkMatchSilence) { + AlertMessageEntity alertCache = JSONObject.parseObject((String)silences.get(alertRecv.getHashKey()),AlertMessageEntity.class); + log.debug("缓存中静默规则匹配 值为 :"+JSON.toJSONString(alertCache)); + if(ObjectUtil.isEmpty(alertCache)) { + // 无缓存则 判断库表是否有值 更新状态 加到缓存中去 + AlertMessageEntity alertMessageDb = alertHashKeyAndEntityMap.get(alertRecv.getHashKey()); + log.debug("数据库中的告警数据为:"+JSON.toJSONString(alertMessageDb)); + if(ObjectUtil.isEmpty(alertMessageDb)) { + alertRecv.setState(4); + entities.add(alertRecv); + }else { + // 修改数据库中的状态 + if(!alertMessageDb.getState().equals(4)) { + alertMessageDb.setState(4); + entities.add(alertMessageDb); + } + } + // 存入缓存 + redisTemplate.opsForHash().put(Constant.ALERT_MESSAGE_SILENCE, alertRecv.getHashKey(), JSON.toJSONString(alertRecv)); + }else { + continue; + } + }else { + // 未匹配静默 则判断状态 新增、持续、失效告警 + // 如果是内置告警直接入库 + if (buildRuleId.equals(alertRecv.getRuleId())) { + log.debug("新增内置规则数据直接入库"); + entities.add(alertRecv); + } else { + Date endAt = alertRecv.getEndAt(); + // 当前时间比结束时间早 告警激活入库 + if (currentTime.before(endAt)) { + log.debug("新增或持续告警信息 唯一标识为:"+alertRecv.getHashKey()); + // 已存在告警则不保存入库 + if (alertHashKeyAndEntityMap.get(alertRecv.getHashKey()) != null) continue; + alertRecv.setState(1); + alertRecv.setEndAt(null); + entities.add(alertRecv); + } else { + log.debug("失效告警信息 唯一标识为:"+alertRecv.getHashKey()); + // 当前时间比结束时间晚 告警失效 + AlertMessageEntity expiredAlert = alertHashKeyAndEntityMap.get(alertRecv.getHashKey()); + // 消息为空或者消息已经过期则不进行操作 + if(expiredAlert == null || expiredAlert.getState().equals(2)) continue; + expiredAlert.setState(2); + expiredAlert.setEndAt(alertRecv.getEndAt()); + entities.add(expiredAlert); + } + } + } + }catch(Exception e) { + e.printStackTrace(); + log.error("取出redis中接收告警信息 转换异常:"+e.getMessage()); + continue; + } + } + + if(ObjectUtils.isNotEmpty(entities)) { + alertMessageService.saveOrUpdateBatch(entities); + log.debug("入库告警信息为:"+JSON.toJSONString(entities)); + //新增以及失效的告警信息存入邮件发送缓存中 + redisTemplate.opsForList().leftPush(Constant.ALERT_MESSAGE_NOTIFY, JSON.toJSONString(entities)); + }else { + log.debug("暂无告警消息入库"); + } + } + } + }); + + executorService.execute(new Runnable() { + + @Override + public void run() { + log.debug("告警信息发送线程运行"); + while(true) { + try { + // 加载模板文件 是否匹配silence 发送告警通知 记录通知结果 + template = this.getTemplate(); + this.execAlertMessage(); + }catch (ParseException e) { + log.error("告警通知异常",e); + } catch (MalformedTemplateNameException e) { + log.error("告警通知异常",e); + } catch (TemplateNotFoundException e) { + log.error("告警通知异常",e); + } catch (IOException e) { + log.error("告警通知异常",e); + } + } + } + + private void execAlertMessage() { + ListOperations<String, Object> ops = redisTemplate.opsForList(); + while(true) { + Object obj = ops.rightPop(Constant.ALERT_MESSAGE_NOTIFY); + if (ToolUtil.isEmpty(obj)) { + continue; + } + // 告警消息集合 + List<AlertMessageEntity> entities = null; + try { + entities = JSONObject.parseArray(obj.toString(), AlertMessageEntity.class); + } catch (Exception e) { + log.error("从redis中取告警消息转化失败,错误信息:" +e + " 转化参数是:" + obj.toString()); + } + log.info("取出告警消息,准备邮件通知:" + JSONObject.toJSONString(entities)); + if(ObjectUtil.isEmpty(entities)){ + continue; + } + + // 判断是否匹配静默规则 + for(int i=entities.size()-1;i>=0;i--) { + AlertMessageEntity entity = entities.get(i); + boolean checkMatchSilence = alertSilenceService.checkMatchSilence(entity); + if(checkMatchSilence) { + entities.remove(i); + } + } + + // 匹配完静默判断是否还有告警数据 + if(CollectionUtils.isEmpty(entities)){ + continue; + } + + // 规则及通知人信息 + List<AlertRuleEntity> ruleList = alertRuleService.list(); + Map<Integer, String> ruleIdAndReceiver = ruleList.stream().filter(entity -> ToolUtil.isNotEmpty(entity.getReceiver())).collect(Collectors.toMap(AlertRuleEntity::getId, AlertRuleEntity::getReceiver)); + + // 用户及邮箱信息 + List<SysUserEntity> userList = sysUserService.list(); + Map<Long, String> userIdAndEmail = userList.stream().filter(sysUserEntity -> ToolUtil.isNotEmpty(sysUserEntity.getEmail())).collect(Collectors.toMap(SysUserEntity::getUserId, SysUserEntity::getEmail)); + + // 用户和关联脚本信息 + List<SysUserNotification> userNotificationList = userNotificationDao.selectSysUserNotificationInfo(); + Map<Integer, List<SysUserNotification>> userIdAndNotificationMap = userNotificationList.stream().collect(Collectors.groupingBy(SysUserNotification::getUserId)); + + // 脚本信息 + List<AlertNotificationScript> scriptList = alertNotifiScriptDao.selectList(null); + Map<Integer, String> scriptIdAndFilePathMap = scriptList.stream().collect(Collectors.toMap(AlertNotificationScript::getId, AlertNotificationScript::getFilePath)); + + Map<Integer,List<AlertNotifyLog>> alertNotifyLogMap = new HashMap<Integer,List<AlertNotifyLog>>(); + for (AlertMessageEntity messageEntity : entities) { + String receiver = ruleIdAndReceiver.get(messageEntity.getRuleId()); + // 通知人不为空 发件 + if(StringUtils.isNotEmpty(receiver)){ + + log.info("邮件关联用户,进行邮件通知"); + List<String> toSendEmails = new ArrayList<>(); + + // 用户关联脚本 + List<SysUserNotification> userNotifications = new ArrayList<>(); + + // 发送记录 + List<AlertNotifyLog> alertNotifyLogs = new ArrayList<AlertNotifyLog>(); + + Arrays.asList(receiver.split(",")).stream().forEach(receiverId -> { + String email = userIdAndEmail.get(Long.valueOf(receiverId)); + if(StringUtils.isNotEmpty(email)){ + toSendEmails.add(email); + } + + AlertNotifyLog alertNotifyLog = new AlertNotifyLog(); + alertNotifyLog.setMessageId(messageEntity.getId()); + alertNotifyLog.setMessageState(messageEntity.getState()); + alertNotifyLog.setMethod(email); + alertNotifyLog.setUserId(Long.valueOf(receiverId)); + alertNotifyLogs.add(alertNotifyLog); + alertNotifyLogMap.put(messageEntity.getId(), alertNotifyLogs); + + // 通过关联用户 + List<SysUserNotification> sysUserNotifications = userIdAndNotificationMap.get(Integer.valueOf(receiverId)); + if(CollectionUtils.isNotEmpty(sysUserNotifications)){ + userNotifications.addAll(sysUserNotifications); + } + }); + + log.info("需要发送的邮箱 -> {}",JSON.toJSONString(toSendEmails)); + String state = "UNKNOWN"; + switch (messageEntity.getState()) { + case 1:state = "Pending";break; + case 2:state = "Expired";break; + } + + if (ToolUtil.isNotEmpty(toSendEmails)) { + try { + mailUtil.sendHTML(toSendEmails, state + " : " + messageEntity.getSummary(), this.sendMail(messageEntity)); + List<AlertNotifyLog> list = alertNotifyLogMap.get(messageEntity.getId()); + if(ObjectUtil.isNotEmpty(list)) { + for(AlertNotifyLog log : list) { + log.setState(1); + } + } + } catch (Exception e) { + List<AlertNotifyLog> list = alertNotifyLogMap.get(messageEntity.getId()); + if(ObjectUtil.isNotEmpty(list)) { + for(AlertNotifyLog log : list) { + log.setState(0); + log.setErrorMsg(e.getMessage()); + } + } + log.error("邮件发送任务异常",e); + } + } + + // 通知脚本信息不为空 + if (CollectionUtils.isNotEmpty(userNotifications)) { + log.info("邮件进行脚本通知"); + executorService.submit(() -> this.notifiScript(userNotifications, scriptIdAndFilePathMap, messageEntity)); + } else { + log.info("邮件没有关联用户脚本,不进行脚本通知"); + } + }else{ + log.info("邮件没有关联用户,不进行邮件通知"); + } + } + + List<AlertNotifyLog> alertNotifyLogToDb =new ArrayList<AlertNotifyLog>(); + // 将发送记录批量存入库中 + for(List<AlertNotifyLog> data:alertNotifyLogMap.values()) { + alertNotifyLogToDb.addAll(data); + } + alertNotifyLogService.saveBatch(alertNotifyLogToDb); + } + } + + private void notifiScript(List<SysUserNotification> userNotifications, Map<Integer, String> scriptIdAndFilePathMap, AlertMessageEntity alertMsg) { + List<AlertNotifyLog> alertNotifyLogs =new ArrayList<AlertNotifyLog>(); + + String filePath, account; + for (SysUserNotification userNotification : userNotifications) { + + AlertNotifyLog alertNotifyLog =new AlertNotifyLog(); + alertNotifyLog.setMessageId(alertMsg.getId()); + alertNotifyLog.setMessageState(alertMsg.getState()); + alertNotifyLog.setMethod(userNotification.getScriptName()); + alertNotifyLog.setUserId(Long.valueOf(userNotification.getUserId())); + + filePath = scriptIdAndFilePathMap.get(userNotification.getScriptId()); + account = userNotification.getAccount(); + String[] commands = new String[]{filePath, account, JSONObject.toJSONString(alertMsg)}; + try { + String result = RuntimeUtil.execForStr(commands); + if (log.isDebugEnabled()) + log.debug(String.format("通知命令执行结束,执行命令是: %s , 结果是: %s", Arrays.asList(commands).toString(), result)); + alertNotifyLog.setState(1); + } catch (Exception e) { + log.debug(String.format("通知命令执行结束,执行命令是: %s , 结果是: %s", Arrays.asList(commands).toString(), "失败")); + log.error("alert通知脚本异常,异常消息是:" + e); + alertNotifyLog.setState(0); + alertNotifyLog.setErrorMsg(e.getMessage()); + } + alertNotifyLogs.add(alertNotifyLog); + } + alertNotifyLogService.saveBatch(alertNotifyLogs); + } + + public String sendMail(AlertMessageEntity alertMessage) throws IOException, TemplateException { + //创建模型数据 + Map<String,Object> modelMap = new HashMap<String,Object>(); + modelMap.put("alertMessage",alertMessage); + //执行静态化 方式1 获取静态化内容 + String content = FreeMarkerTemplateUtils.processTemplateIntoString(template, modelMap); + //静态化内容; + return content; + } + + private Template getTemplate() throws TemplateNotFoundException, MalformedTemplateNameException, ParseException, IOException { + String templateString = configService.getValue("email_template"); + // 创建配置类 + Configuration configuration = new Configuration(Configuration.getVersion()); + //创建模板加载器 + StringTemplateLoader templateLoader = new StringTemplateLoader(); + // 存入模板 + templateLoader.putTemplate("template", templateString); //template = 虚拟名称, 用来当作获取静态文件的key + //加载模板加载器 + configuration.setTemplateLoader(templateLoader); + //得到模板 + Template template = configuration.getTemplate("template", "utf-8"); + return template; + } + }); + } +}
\ No newline at end of file diff --git a/nz-admin/src/main/java/com/nis/common/config/SettingConfig.java b/nz-admin/src/main/java/com/nis/common/config/SettingConfig.java deleted file mode 100644 index be2106a3..00000000 --- a/nz-admin/src/main/java/com/nis/common/config/SettingConfig.java +++ /dev/null @@ -1,21 +0,0 @@ -package com.nis.common.config; - -import com.nis.modules.sys.dao.SysConfigDao; -import lombok.extern.slf4j.Slf4j; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.boot.CommandLineRunner; - -@Slf4j -//@Component -public class SettingConfig implements CommandLineRunner { - - @Autowired - private SysConfigDao sysConfigDao; - - @Override - public void run(String... args) throws Exception { - /* SysConfigEntity sysConfigEntity = sysConfigDao.selectOne(new QueryWrapper<SysConfigEntity>().lambda().eq(SysConfigEntity::getParamKey, "query_max_series")); - PromConstant.queryMaxSeries = Integer.valueOf(sysConfigEntity.getParamValue()); - log.info("当前 Prometheus query_range 查询最大条数限制是 " + sysConfigEntity.getParamValue());*/ - } -}
\ No newline at end of file diff --git a/nz-admin/src/main/java/com/nis/modules/alert/controller/AlertMessageReceiveController.java b/nz-admin/src/main/java/com/nis/modules/alert/controller/AlertMessageReceiveController.java index de06ff36..c7338e21 100644 --- a/nz-admin/src/main/java/com/nis/modules/alert/controller/AlertMessageReceiveController.java +++ b/nz-admin/src/main/java/com/nis/modules/alert/controller/AlertMessageReceiveController.java @@ -39,7 +39,7 @@ public class AlertMessageReceiveController { @PostMapping public R alerts(@RequestBody String receiveStr, HttpServletRequest request){ try { - alertMessageService.saveBatch(receiveStr, request); + alertMessageService.saveBatchToCache(receiveStr, request); } catch (Exception e) { logger.error(request.getRemoteAddr() + " 推送过来的 promethues 告警消息处理异常,消息是:" + receiveStr + " 异常信息是:" + e); throw new NZException(RCode.ALERTMSG_PARSE_ERROR); diff --git a/nz-admin/src/main/java/com/nis/modules/alert/dao/AlertNotifyLogDao.java b/nz-admin/src/main/java/com/nis/modules/alert/dao/AlertNotifyLogDao.java new file mode 100644 index 00000000..4116d929 --- /dev/null +++ b/nz-admin/src/main/java/com/nis/modules/alert/dao/AlertNotifyLogDao.java @@ -0,0 +1,13 @@ +package com.nis.modules.alert.dao; + + + +import org.apache.ibatis.annotations.Mapper; +import com.baomidou.mybatisplus.core.mapper.BaseMapper; +import com.nis.modules.alert.entity.AlertNotifyLog; + +@Mapper +public interface AlertNotifyLogDao extends BaseMapper<AlertNotifyLog>{ + + +} diff --git a/nz-admin/src/main/java/com/nis/modules/alert/dao/AlertSilenceDao.java b/nz-admin/src/main/java/com/nis/modules/alert/dao/AlertSilenceDao.java index 3eda52bc..6412bd44 100644 --- a/nz-admin/src/main/java/com/nis/modules/alert/dao/AlertSilenceDao.java +++ b/nz-admin/src/main/java/com/nis/modules/alert/dao/AlertSilenceDao.java @@ -8,10 +8,13 @@ import org.apache.ibatis.annotations.Param; import com.baomidou.mybatisplus.core.mapper.BaseMapper; import com.baomidou.mybatisplus.core.metadata.IPage; +import com.nis.modules.alert.entity.AlertMessageEntity; import com.nis.modules.alert.entity.AlertSilenceConf; @Mapper public interface AlertSilenceDao extends BaseMapper<AlertSilenceConf>{ List<AlertSilenceConf> selectAlertSilenceConfs(IPage<AlertSilenceConf> page,@Param("params")Map<String,Object> params); + + void updateSilenceState(); } diff --git a/nz-admin/src/main/java/com/nis/modules/alert/entity/AlertNotifyLog.java b/nz-admin/src/main/java/com/nis/modules/alert/entity/AlertNotifyLog.java new file mode 100644 index 00000000..758420f8 --- /dev/null +++ b/nz-admin/src/main/java/com/nis/modules/alert/entity/AlertNotifyLog.java @@ -0,0 +1,25 @@ +package com.nis.modules.alert.entity; + + +import com.baomidou.mybatisplus.annotation.TableName; + +import lombok.Data; + +@Data +@TableName("alert_notify_log") +public class AlertNotifyLog { + + private Integer id; + + private Integer messageId; + + private Long userId; + + private String method; + + private Integer messageState; + + private Integer state; + + private String errorMsg; +} diff --git a/nz-admin/src/main/java/com/nis/modules/alert/job/AlertMessageJobConfig.java b/nz-admin/src/main/java/com/nis/modules/alert/job/AlertMessageJobConfig.java index a65fb85e..2efd4f0a 100644 --- a/nz-admin/src/main/java/com/nis/modules/alert/job/AlertMessageJobConfig.java +++ b/nz-admin/src/main/java/com/nis/modules/alert/job/AlertMessageJobConfig.java @@ -20,6 +20,9 @@ public class AlertMessageJobConfig { @Value("${nezha.alertMessageJobCron}") private String alertMessageJobCron; + @Value("${nezha.alertSilenceJobCron}") + private String alertSilenceJobCron; + @Autowired private SchedulerFactoryBean schedulerFactoryBean; @@ -28,6 +31,7 @@ public class AlertMessageJobConfig { private String alertMessageEmailNotificationJobName = "ALERTMESSAGEEMAILNOTIFICATION_JOB"; private String alertMessageExpirStatusJobName = "ALERTMESSAGEEXPIRSTATUS_JOB"; + private String alertSilenceStatusJobName = "ALERTSILENCESTATUS_JOB"; @Bean public JobDetail alertMessageEmailNotifiJobDetail() { @@ -38,6 +42,11 @@ public class AlertMessageJobConfig { public JobDetail alertMessageExpirStatusJobDetail() { return JobBuilder.newJob(AlertMessageExpirStatusJob.class).withIdentity(ScheduleUtils.JOB_NAME + alertMessageExpirStatusJobName).storeDurably().build(); } + + @Bean + public JobDetail alertSilenceStatusJobDetail() { + return JobBuilder.newJob(AlertSilenceStatusJob.class).withIdentity(ScheduleUtils.JOB_NAME + alertSilenceStatusJobName).storeDurably().build(); + } @PostConstruct public void init() throws Exception { @@ -55,5 +64,8 @@ public class AlertMessageJobConfig { // alertmessageJob 配置 ScheduleUtils.createCronScheduleJob(alertMessageExpirStatusJobName, alertMessageExpirStatusJobDetail(), alertMessageJobCron, scheduler); + + // alertSilenceStatusJob 配置 + ScheduleUtils.createCronScheduleJob(alertSilenceStatusJobName, alertSilenceStatusJobDetail(), alertSilenceJobCron, scheduler); } } diff --git a/nz-admin/src/main/java/com/nis/modules/alert/job/AlertSilenceStatusJob.java b/nz-admin/src/main/java/com/nis/modules/alert/job/AlertSilenceStatusJob.java new file mode 100644 index 00000000..2c66d2d5 --- /dev/null +++ b/nz-admin/src/main/java/com/nis/modules/alert/job/AlertSilenceStatusJob.java @@ -0,0 +1,61 @@ +package com.nis.modules.alert.job; + +import java.util.Arrays; +import java.util.List; +import java.util.Map; +import org.quartz.DisallowConcurrentExecution; +import org.quartz.JobExecutionContext; +import org.quartz.JobExecutionException; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.data.redis.core.RedisTemplate; +import org.springframework.scheduling.quartz.QuartzJobBean; +import com.alibaba.fastjson.JSONObject; +import com.baomidou.mybatisplus.core.conditions.query.QueryWrapper; +import com.nis.common.utils.Constant; +import com.nis.modules.alert.entity.AlertMessageEntity; +import com.nis.modules.alert.entity.AlertSilenceConf; +import com.nis.modules.alert.service.AlertSilenceService; +import cn.hutool.core.util.ObjectUtil; +import cn.hutool.log.Log; + +@DisallowConcurrentExecution +public class AlertSilenceStatusJob extends QuartzJobBean{ + + private static final Log log = Log.get(); + + @Autowired + private AlertSilenceService alertSilenceService; + + @Autowired + private RedisTemplate<String,Object> redisTemplate; + + @Override + protected void executeInternal(JobExecutionContext arg0) throws JobExecutionException { + log.debug("静默状态定时任务开始"); + + Integer[] states= {1,2}; + // 将静默配置状态为active和pending的判断是否变更状态 + List<AlertSilenceConf> alertSilenceConfs = alertSilenceService.list(new QueryWrapper<AlertSilenceConf>().lambda().in(AlertSilenceConf::getState, Arrays.asList(states))); + + if(ObjectUtil.isEmpty(alertSilenceConfs)) { + log.debug("没有任何静默配置信息"); + redisTemplate.delete(Constant.ALERT_MESSAGE_SILENCE); + log.debug("清空缓存里的数据信息"); + return; + }else { + //修改silence状态 + alertSilenceService.modifySilenceState(); + //清除缓存中废弃的数据 + Map<Object, Object> silences = redisTemplate.opsForHash().entries(Constant.ALERT_MESSAGE_SILENCE); + for (Map.Entry<Object, Object> entry : silences.entrySet()) { + AlertMessageEntity alertMessage = JSONObject.parseObject((String)entry.getValue(),AlertMessageEntity.class); + boolean checkMatchSilence = alertSilenceService.checkMatchSilence(alertMessage); + if(checkMatchSilence) { + continue; + }else { + redisTemplate.opsForHash().delete(Constant.ALERT_MESSAGE_SILENCE,entry.getKey()); + } + } + } + } +} diff --git a/nz-admin/src/main/java/com/nis/modules/alert/service/AlertMessageService.java b/nz-admin/src/main/java/com/nis/modules/alert/service/AlertMessageService.java index 111f2761..74689091 100644 --- a/nz-admin/src/main/java/com/nis/modules/alert/service/AlertMessageService.java +++ b/nz-admin/src/main/java/com/nis/modules/alert/service/AlertMessageService.java @@ -34,5 +34,7 @@ public interface AlertMessageService extends IService<AlertMessageEntity> { void updateAlertStateByIds(Map<String, Object> params); List<MetricsDto> queryAlertMessageStatisticsInfoByRule(); + + void saveBatchToCache(String receiveStr, HttpServletRequest request) throws Exception; } diff --git a/nz-admin/src/main/java/com/nis/modules/alert/service/AlertNotifyLogService.java b/nz-admin/src/main/java/com/nis/modules/alert/service/AlertNotifyLogService.java new file mode 100644 index 00000000..12046b75 --- /dev/null +++ b/nz-admin/src/main/java/com/nis/modules/alert/service/AlertNotifyLogService.java @@ -0,0 +1,10 @@ +package com.nis.modules.alert.service; + + +import com.baomidou.mybatisplus.extension.service.IService; +import com.nis.modules.alert.entity.AlertNotifyLog; + +public interface AlertNotifyLogService extends IService<AlertNotifyLog>{ + + +} diff --git a/nz-admin/src/main/java/com/nis/modules/alert/service/AlertSilenceService.java b/nz-admin/src/main/java/com/nis/modules/alert/service/AlertSilenceService.java index bc3f2da1..2e8445ce 100644 --- a/nz-admin/src/main/java/com/nis/modules/alert/service/AlertSilenceService.java +++ b/nz-admin/src/main/java/com/nis/modules/alert/service/AlertSilenceService.java @@ -5,6 +5,7 @@ import java.util.Map; import com.baomidou.mybatisplus.extension.service.IService; import com.nis.common.utils.PageUtils; +import com.nis.modules.alert.entity.AlertMessageEntity; import com.nis.modules.alert.entity.AlertSilenceConf; public interface AlertSilenceService extends IService<AlertSilenceConf>{ @@ -16,4 +17,8 @@ public interface AlertSilenceService extends IService<AlertSilenceConf>{ void removeByIds(List<Integer> ids); AlertSilenceConf queryAlertSilence(Integer id); + + boolean checkMatchSilence(AlertMessageEntity alertMessageEntity); + + void modifySilenceState(); } diff --git a/nz-admin/src/main/java/com/nis/modules/alert/service/impl/AlertMessageServiceImpl.java b/nz-admin/src/main/java/com/nis/modules/alert/service/impl/AlertMessageServiceImpl.java index 8a62e5b8..05cbea9c 100644 --- a/nz-admin/src/main/java/com/nis/modules/alert/service/impl/AlertMessageServiceImpl.java +++ b/nz-admin/src/main/java/com/nis/modules/alert/service/impl/AlertMessageServiceImpl.java @@ -46,6 +46,7 @@ import org.apache.poi.xssf.streaming.SXSSFWorkbook; import org.apache.shiro.SecurityUtils; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.data.redis.core.HashOperations; +import org.springframework.data.redis.core.ListOperations; import org.springframework.data.redis.core.RedisTemplate; import org.springframework.stereotype.Service; import org.springframework.transaction.annotation.Transactional; @@ -571,4 +572,71 @@ public class AlertMessageServiceImpl extends ServiceImpl<AlertMessageDao, AlertM ExcelUtils.setDataCellStyle(dataStyle, r, 13); } } + + @Override + public void saveBatchToCache(String receiveStr, HttpServletRequest request) throws Exception { + logger.debug(request.getRemoteAddr() + " 推送过来的 promethues 告警消息是 {}", receiveStr); + + // 接收告警的当前时间 + Date currentTime = DateUtil.getUTCTimeByConfigTimeZone(); + + List<AlertMessageReceiveDTO> receiveData = JSONObject.parseArray(receiveStr, AlertMessageReceiveDTO.class); + if (CollectionUtils.isEmpty(receiveData)) return; + + AlertRuleEntity buildRule = alertRuleService.getOne(new QueryWrapper<AlertRuleEntity>().lambda().eq(AlertRuleEntity::getBuildIn, 1)); + Integer buildRuleId = buildRule.getId(); + + // 当前激活状态的告警消息 + List<AlertMessageEntity> pendingAlert = this.list(new QueryWrapper<AlertMessageEntity>().lambda().eq(AlertMessageEntity::getState, 1)); + Map<String, AlertMessageEntity> alertHashKeyAndEntityMap = new HashMap<>(); + for (AlertMessageEntity entity : pendingAlert) { + alertHashKeyAndEntityMap.put(entity.getHashKey(), entity); + } + + List<Endpoint> endpointList = endpointService.list(); + Map<Integer, Endpoint> endpointMap = new HashMap<>(); + for (Endpoint endpoint : endpointList) { + endpointMap.put(endpoint.getId(), endpoint); + } + + List<Module> moduleList = moduleService.list(); + Map<Integer, Module> moduleIdAndEntityMap = new HashMap<>(); + Map<String, Module> moduleNameAndEntityMap = new HashMap<>(); + for (Module module : moduleList) { + moduleIdAndEntityMap.put(module.getId(), module); + moduleNameAndEntityMap.put(module.getName(), module); + } + + List<Asset> assetList = assetService.list(); + Map<Integer, Asset> assetIdAndEntityMap = new HashMap<>(); + Map<String, Asset> assetHostAndEntityMap = new HashMap<>(); + for (Asset asset : assetList) { + assetIdAndEntityMap.put(asset.getId(), asset); + assetHostAndEntityMap.put(asset.getHost(), asset); + } + + List<Project> projectList = projectService.list(); + Map<String, Integer> projectNameAndIdMap = projectList.stream().collect(Collectors.toMap(Project::getName, Project::getId)); + + List<Idc> idcList = idcService.list(); + Map<String, Integer> idcNameAndIdMap = idcList.stream().collect(Collectors.toMap(Idc::getName, Idc::getId)); + + AlertMessageEntity entity; + List<AlertMessageEntity> entities =new ArrayList<AlertMessageEntity>(); + ListOperations<String, Object> opsForList = redisTemplate.opsForList(); + + for (AlertMessageReceiveDTO dto : receiveData) { + try { + getReceiveLabels(dto, endpointMap, assetIdAndEntityMap, assetHostAndEntityMap, projectNameAndIdMap, idcNameAndIdMap, moduleIdAndEntityMap, moduleNameAndEntityMap); + entity = AlertMessageReceiveDTO.toAlertMessageEntity(dto, buildRuleId); + entities.add(entity); + + } catch (Exception e) { + logger.error("解析告警信息失败,参数是:" + JSONObject.toJSONString(dto) + " 错误信息:" ,e); + throw new NZException(RCode.ALERTMSG_SAVE_ERROR); + } + } + // 存入缓存 + opsForList.leftPush(Constant.ALERT_MESSAGE_RECV, JSON.toJSONString(entities)); + } }
\ No newline at end of file diff --git a/nz-admin/src/main/java/com/nis/modules/alert/service/impl/AlertNotifyLogServiceImpl.java b/nz-admin/src/main/java/com/nis/modules/alert/service/impl/AlertNotifyLogServiceImpl.java new file mode 100644 index 00000000..a70277f3 --- /dev/null +++ b/nz-admin/src/main/java/com/nis/modules/alert/service/impl/AlertNotifyLogServiceImpl.java @@ -0,0 +1,13 @@ +package com.nis.modules.alert.service.impl; + +import org.springframework.stereotype.Service; + +import com.baomidou.mybatisplus.extension.service.impl.ServiceImpl; +import com.nis.modules.alert.dao.AlertNotifyLogDao; +import com.nis.modules.alert.entity.AlertNotifyLog; +import com.nis.modules.alert.service.AlertNotifyLogService; + +@Service("alertNotifyLogService") +public class AlertNotifyLogServiceImpl extends ServiceImpl<AlertNotifyLogDao,AlertNotifyLog> implements AlertNotifyLogService{ + +} diff --git a/nz-admin/src/main/java/com/nis/modules/alert/service/impl/AlertSilenceServiceImpl.java b/nz-admin/src/main/java/com/nis/modules/alert/service/impl/AlertSilenceServiceImpl.java index c2621d43..e961f0d7 100644 --- a/nz-admin/src/main/java/com/nis/modules/alert/service/impl/AlertSilenceServiceImpl.java +++ b/nz-admin/src/main/java/com/nis/modules/alert/service/impl/AlertSilenceServiceImpl.java @@ -10,6 +10,7 @@ import org.apache.commons.lang3.time.DateUtils; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Service; +import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper; import com.baomidou.mybatisplus.core.conditions.query.QueryWrapper; import com.baomidou.mybatisplus.core.metadata.IPage; import com.baomidou.mybatisplus.core.toolkit.ObjectUtils; @@ -22,6 +23,7 @@ import com.nis.common.utils.RCode; import com.nis.modules.alert.dao.AlertMessageDao; import com.nis.modules.alert.dao.AlertRuleDao; import com.nis.modules.alert.dao.AlertSilenceDao; +import com.nis.modules.alert.entity.AlertMessageEntity; import com.nis.modules.alert.entity.AlertRuleEntity; import com.nis.modules.alert.entity.AlertSilenceConf; import com.nis.modules.alert.service.AlertSilenceService; @@ -217,5 +219,52 @@ public class AlertSilenceServiceImpl extends ServiceImpl<AlertSilenceDao,AlertSi } return result; } + + @Override + public boolean checkMatchSilence(AlertMessageEntity alertMessageEntity) { + if(ObjectUtil.isEmpty(alertMessageEntity)) { + return false; + } + List<AlertSilenceConf> alertSilenceConfs = this.list(new LambdaQueryWrapper<AlertSilenceConf>().eq(AlertSilenceConf::getRuleId, alertMessageEntity.getRuleId()).eq(AlertSilenceConf::getState, 2)); + if(ObjectUtil.isEmpty(alertSilenceConfs)) { + return false; + }else { + for(AlertSilenceConf alertSilenceConf : alertSilenceConfs) { + if(alertSilenceConf.getLinkId().equals(-1)) { + return true; + } + switch(alertSilenceConf.getType()) { + case "datacenter": + if(alertSilenceConf.getLinkId().equals(alertMessageEntity.getDcId())){ + return true; + } + case "project": + if(alertSilenceConf.getLinkId().equals(alertMessageEntity.getProjectId())){ + return true; + } + case "module": + if(alertSilenceConf.getLinkId().equals(alertMessageEntity.getModuleId())){ + return true; + } + case "endpoint": + if(alertSilenceConf.getLinkId().equals(alertMessageEntity.getEndpointId())){ + return true; + } + case "asset": + if(alertSilenceConf.getLinkId().equals(alertMessageEntity.getAssetId())){ + return true; + } + } + } + } + return false; + } + + @Override + public void modifySilenceState() { + + alertSilenceDao.updateSilenceState(); + + } } diff --git a/nz-admin/src/main/java/com/nis/modules/sys/dao/SysUserNotificationDao.java b/nz-admin/src/main/java/com/nis/modules/sys/dao/SysUserNotificationDao.java index f5bd6ae1..7fe674bc 100644 --- a/nz-admin/src/main/java/com/nis/modules/sys/dao/SysUserNotificationDao.java +++ b/nz-admin/src/main/java/com/nis/modules/sys/dao/SysUserNotificationDao.java @@ -2,9 +2,14 @@ package com.nis.modules.sys.dao; import com.baomidou.mybatisplus.core.mapper.BaseMapper; import com.nis.modules.sys.entity.SysUserNotification; + +import java.util.List; + import org.apache.ibatis.annotations.Mapper; @Mapper public interface SysUserNotificationDao extends BaseMapper<SysUserNotification> { - + + List<SysUserNotification> selectSysUserNotificationInfo(); + } diff --git a/nz-admin/src/main/java/com/nis/modules/sys/entity/SysUserNotification.java b/nz-admin/src/main/java/com/nis/modules/sys/entity/SysUserNotification.java index 540d4bac..1960b4af 100644 --- a/nz-admin/src/main/java/com/nis/modules/sys/entity/SysUserNotification.java +++ b/nz-admin/src/main/java/com/nis/modules/sys/entity/SysUserNotification.java @@ -1,5 +1,6 @@ package com.nis.modules.sys.entity; +import com.baomidou.mybatisplus.annotation.TableField; import com.baomidou.mybatisplus.annotation.TableId; import com.baomidou.mybatisplus.annotation.TableName; import lombok.Data; @@ -21,4 +22,6 @@ public class SysUserNotification implements Serializable { private String account; + @TableField(exist = false) + private String scriptName; }
\ No newline at end of file diff --git a/nz-admin/src/main/resources/application.yml b/nz-admin/src/main/resources/application.yml index 9db905b8..44e92e7a 100644 --- a/nz-admin/src/main/resources/application.yml +++ b/nz-admin/src/main/resources/application.yml @@ -53,6 +53,7 @@ nezha: promserverJobCron: 0/15 * * * * ? assetPingJobCron: 0 0/3 * * * ? * # asset ping job 同步程序执行频率 不设置默认情况为3分钟 assetFeatureJobCron: 0 0 1 * * ? # asset采集信息 默认5分钟执行一次 + alertSilenceJobCron: 0/5 * * * * ? # 每5秒检测一次静默配置 loginExpiration: 120 #登录token 过期时间,单位 分钟 ,不设置默认为30分钟 corePoolSize: 20 maxPoolSize: 100 diff --git a/nz-admin/src/main/resources/db/V2021.03.16__1.Insert alert_notification_log table.sql b/nz-admin/src/main/resources/db/V2021.03.16__1.Insert alert_notification_log table.sql new file mode 100644 index 00000000..f7335d04 --- /dev/null +++ b/nz-admin/src/main/resources/db/V2021.03.16__1.Insert alert_notification_log table.sql @@ -0,0 +1,19 @@ +SET NAMES utf8mb4; +SET FOREIGN_KEY_CHECKS = 0; + +-- ---------------------------- +-- Table structure for alert_notify_log +-- ---------------------------- +DROP TABLE IF EXISTS `alert_notify_log`; +CREATE TABLE `alert_notify_log` ( + `id` int(10) NOT NULL AUTO_INCREMENT, + `message_id` int(10) NOT NULL COMMENT '关联 alert_message.id', + `user_id` int(10) NOT NULL COMMENT '关联 sys_user.id', + `method` varchar(128) CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci NULL DEFAULT NULL COMMENT '通知方式 VARCHARemail 或 NOTIFICATION_SCRIPT.name', + `message_state` char(1) CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci NULL DEFAULT NULL COMMENT '消息状态1: active2: expired', + `state` char(1) CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci NULL DEFAULT NULL COMMENT '通知状态0:失败1:成功', + `error_msg` varchar(1024) CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci NULL DEFAULT NULL COMMENT '发送错误信息', + PRIMARY KEY (`id`) USING BTREE +) ENGINE = InnoDB AUTO_INCREMENT = 1 CHARACTER SET = utf8mb4 COLLATE = utf8mb4_general_ci ROW_FORMAT = Dynamic; + +SET FOREIGN_KEY_CHECKS = 1;
\ No newline at end of file diff --git a/nz-admin/src/main/resources/mapper/alert/AlertSilenceDao.xml b/nz-admin/src/main/resources/mapper/alert/AlertSilenceDao.xml index 04aeacb5..f2ccd969 100644 --- a/nz-admin/src/main/resources/mapper/alert/AlertSilenceDao.xml +++ b/nz-admin/src/main/resources/mapper/alert/AlertSilenceDao.xml @@ -68,6 +68,9 @@ <if test="params.id != null and params.id !='' "> and asic.id = #{params.id} </if> + <if test="params.state != null and params.state !='' "> + and asic.state = #{params.state} + </if> <if test="params.startAt != null"> and asic.start_at >= #{params.startAt} </if> @@ -99,4 +102,9 @@ </select> + + <update id="updateSilenceState"> + UPDATE alert_silence_conf + SET state = ( CASE WHEN now() < start_at THEN 1 WHEN ( now() >= start_at AND now() <= end_at ) THEN 2 ELSE 3 END); + </update> </mapper>
\ No newline at end of file diff --git a/nz-admin/src/main/resources/mapper/sys/SysUserNotificationDao.xml b/nz-admin/src/main/resources/mapper/sys/SysUserNotificationDao.xml new file mode 100644 index 00000000..9327ef58 --- /dev/null +++ b/nz-admin/src/main/resources/mapper/sys/SysUserNotificationDao.xml @@ -0,0 +1,22 @@ +<?xml version="1.0" encoding="UTF-8"?> +<!DOCTYPE mapper PUBLIC "-//mybatis.org//DTD Mapper 3.0//EN" "http://mybatis.org/dtd/mybatis-3-mapper.dtd"> +<mapper namespace="com.nis.modules.sys.dao.SysUserNotificationDao"> + + <resultMap type="com.nis.modules.sys.entity.SysUserNotification" id="sysUserNotification"> + <result property="id" column="id"/> + <result property="userId" column="user_id"/> + <result property="scriptId" column="script_id"/> + <result property="account" column="account"/> + <result property="scriptName" column="script_name"/> + </resultMap> + + <select id="selectSysUserNotificationInfo" resultMap="sysUserNotification"> + select un.id, + un.user_id, + un.script_id, + un.account, + ns.name as script_name + from user_notification un left join notification_script ns on un.script_id = ns.id + </select> + +</mapper> |
