diff options
| author | shizhendong <[email protected]> | 2023-03-21 17:25:46 +0800 |
|---|---|---|
| committer | shizhendong <[email protected]> | 2023-03-21 17:25:46 +0800 |
| commit | db6894876dca851b8c80227d276c73bcce179e99 (patch) | |
| tree | 01bf3cbf1ae46be67696d4760e289c85ed0835dc | |
| parent | 0f248b697a3ff2fbc6f04ce5436db1e55b8ff004 (diff) | |
refactor: NEZ-2680 flyway 版本升级模块结构变更,删除没有实际使用的代码
8 files changed, 987 insertions, 1129 deletions
diff --git a/nz-admin/src/main/java/com/nis/common/update/DataFormatConversionInterface.java b/nz-admin/src/main/java/com/nis/common/update/DataFormatConversionInterface.java new file mode 100644 index 00000000..44ccf914 --- /dev/null +++ b/nz-admin/src/main/java/com/nis/common/update/DataFormatConversionInterface.java @@ -0,0 +1,40 @@ +package com.nis.common.update; + +import cn.hutool.db.DbUtil; +import cn.hutool.db.SqlConnRunner; +import cn.hutool.log.Log; + +import java.sql.Connection; + +/** + * Data Format Conversion Interface + * + * @author shizhendong + * @date 2023/03/15 + */ +public interface DataFormatConversionInterface { + + Log log = Log.get(); + + /** + * init Sql Runner + * + * @param connection + * @return + */ + default SqlConnRunner initSqlRunner(Connection connection) { + log.info("[initSqlRunner] [begin]"); + SqlConnRunner sqlConnRunner = DbUtil.newSqlConnRunner(connection); + log.info("[initSqlRunner] [finshed]"); + return sqlConnRunner; + } + + /** + * data Format Conversion + * + * @param connection + * @throws Exception + */ + void dataFormatConversion(Connection connection) throws Exception; + +} diff --git a/nz-admin/src/main/java/com/nis/common/update/VersionUpdateInterface.java b/nz-admin/src/main/java/com/nis/common/update/VersionUpdateInterface.java deleted file mode 100644 index 8f710d2e..00000000 --- a/nz-admin/src/main/java/com/nis/common/update/VersionUpdateInterface.java +++ /dev/null @@ -1,55 +0,0 @@ -package com.nis.common.update; - -import cn.hutool.log.Log; -import com.nis.common.utils.Tool; - -public interface VersionUpdateInterface { - static final Log log = Log.get(); - /** - * 版本升级前操作内容 - * @return - */ - default boolean preUpdate() { - return true; - }; - - /** - * 执行版本升级 - */ - void excute() throws Exception; - - /** - * 版本升级 - * 1、升级前准备工作 - * 2、开始升级 - * 3、释放资源 - */ - default void update() { - try { - boolean pre = preUpdate(); - log.debug("prehandle result: {}", pre); - if (pre) { - excute(); - } - } catch (Exception e) { - onException(e); - } finally { - afterUpdate(); - } - }; - - default void onException(Exception e) { - log.error(e, "version update error,shutdown service"); - // 停止服务 - Tool.RuntimeUtil.exec("systemctl stop nz-web.service"); - } - - - /** - * 升级后操作,可用于释放资源 - */ - default void afterUpdate() { - - }; - -} diff --git a/nz-admin/src/main/java/com/nis/common/update/v3_0_to_3_2/service/V3_2_DataFormatConversionService.java b/nz-admin/src/main/java/com/nis/common/update/v3_0_to_3_2/service/V3_2_DataFormatConversionService.java index 0c4437e7..fcfb2e4d 100644 --- a/nz-admin/src/main/java/com/nis/common/update/v3_0_to_3_2/service/V3_2_DataFormatConversionService.java +++ b/nz-admin/src/main/java/com/nis/common/update/v3_0_to_3_2/service/V3_2_DataFormatConversionService.java @@ -1,45 +1,525 @@ package com.nis.common.update.v3_0_to_3_2.service; -import cn.hutool.db.DbUtil; +import cn.hutool.db.Entity; import cn.hutool.db.SqlConnRunner; +import cn.hutool.db.handler.EntityListHandler; +import cn.hutool.db.handler.StringHandler; +import cn.hutool.db.sql.SqlExecutor; import cn.hutool.log.Log; +import com.alibaba.druid.DbType; +import com.alibaba.druid.sql.SQLUtils; +import com.alibaba.fastjson.JSONObject; +import com.nis.common.exception.NZException; +import com.nis.common.update.DataFormatConversionInterface; +import com.nis.common.utils.Constant; +import com.nis.common.utils.JSONUtil; +import com.nis.common.utils.RCode; +import com.nis.common.utils.Tool; +import org.springframework.stereotype.Service; import java.sql.Connection; +import java.sql.SQLException; +import java.util.*; +import java.util.stream.Collectors; +import java.util.stream.Stream; /** - * V3_2_DataFormatConversionService + * V3_2_DataFormatConversionServiceImpl * * @author shizhendong * @date 2023/03/15 */ -public interface V3_2_DataFormatConversionService { +@Service +public class V3_2_DataFormatConversionService implements DataFormatConversionInterface { - Log log = Log.get(); + private static final Log log = Log.get(); /** - * init Sql Runner + * back up table name + */ + private static final String BACK_UP_TABLE_NAME = "visual_chart_v3_2"; + + private Connection connection; + + private SqlConnRunner sqlRunner; + + private static final String[] LETTERS = new String[]{ + "A", "B", "C", "D", "E", "F", "G", + "H", "I", "J", "K", "L", "M", "N", + "O", "P", "Q", "R", "S", "T", + "U", "V", "W", "X", "Y", "Z"}; + + /** + * before Data Format Conversion + * check backup table is exists + * + * @return + */ + private boolean beforeDataFormatConversion() throws SQLException { + String tableName = SqlExecutor.query(this.connection, "SHOW TABLES LIKE ?", new StringHandler(), BACK_UP_TABLE_NAME); + log.info("[beforeDataFormatConversion] [check the backup table exists] [query result: {}]", tableName); + + if (Tool.StrUtil.isEmpty(tableName)) { + log.warn("[beforeDataFormatConversion] [The backup table does not exist. Do not perform data conversion] [table: {}]", BACK_UP_TABLE_NAME); + return true; + } + + log.info("[beforeDataFormatConversion] [The backup table exists, execute data conversion] [table: {}]", BACK_UP_TABLE_NAME); + return true; + } + + + @Override + public void dataFormatConversion(Connection connection) throws Exception { + // init sql conn runner + this.connection = connection; + sqlRunner = this.initSqlRunner(connection); + + boolean isConversion = this.beforeDataFormatConversion(); + if (!isConversion) { + return; + } + + log.info("[dataFormatConversion] [conversion V3_0 to V3_2 Chart Data]"); + + // handle visual_chart_element name + this.handleElementName(); + + // conversion Visual Chart Data + this.conversionVisualChartData(); + + // remove back up table + log.info("[dataFormatConversion] [remove back up table] [table_name: {}]", BACK_UP_TABLE_NAME); + SqlExecutor.execute(this.connection, Tool.StrUtil.concat(true, "DROP TABLE ", BACK_UP_TABLE_NAME)); + } + + /** + * chart element name 处理 + * 初始化规则: A-Z AA-AZ BA-BZ AAA-AAZ 顺序添加 + */ + private void handleElementName() throws SQLException { + log.info("[handleElementName] [insert chart element name auto] [begin]"); + log.info("[handleElementName] [insert chart element name auto] [rule: {}]", "A-Z AA-AZ BA-BZ AAA-AAZ..."); + + List<Entity> chartElementList = sqlRunner.findAll(this.connection, Entity.create("visual_chart_element")); + Map<Object, List<Entity>> elementGroupingByChartIdMap = chartElementList.stream().collect(Collectors.groupingBy(entity -> entity.get("chart_id"))); + for (Map.Entry<Object, List<Entity>> entry : elementGroupingByChartIdMap.entrySet()) { + List<Entity> elementListForChart = entry.getValue(); + for (int i = 0; i < elementListForChart.size(); i++) { + String letter = this.transformNumToLetter(i); + Entity chartElement = elementListForChart.get(i); + chartElement.set("name", letter); + } + } + List<Entity> allElementList = elementGroupingByChartIdMap.values().stream().flatMap(Collection::stream).collect(Collectors.toList()); + log.info("[handleElementName] [update element size: {}]", allElementList.size()); + if (Tool.CollUtil.isNotEmpty(allElementList)) { + // 防止 sql 过长,分割后拼接 sql 修改 + Integer batchSize = 1000; + Integer limit = (allElementList.size() + batchSize - 1) / batchSize; + List<List<Entity>> splitVisualChartElementList = Stream.iterate(0, n -> n + 1).limit(limit).parallel().map(a -> allElementList.stream().skip(a * batchSize).limit(batchSize).parallel().collect(Collectors.toList())).collect(Collectors.toList()); + + SQLUtils.FormatOption formatOption = new SQLUtils.FormatOption(false, false); + for (List<Entity> batchUpdateElementList : splitVisualChartElementList) { + List<String> list = Tool.ListUtil.list(true); + for (Entity entity : batchUpdateElementList) { + String sql = SQLUtils.format("update visual_chart_element set name = ? where id = ?", DbType.mariadb, Tool.ListUtil.of(entity.getStr("name"), entity.get("id")), formatOption); + list.add(sql); + } + // execute Batch + SqlExecutor.executeBatch(this.connection, list); + } + } + log.info("[handleElementName] [insert chart element name auto] [finshed]"); + } + + /** + * conversion Visual Chart Data + * + * @throws SQLException + */ + private void conversionVisualChartData() throws SQLException { + log.info("[conversionVisualChartData] [begin]"); + + List<Entity> backUpChartList = SqlExecutor.query(this.connection, "SELECT * FROM visual_chart_v3_2 ORDER BY panel_id, group_id, weight", new EntityListHandler()); + log.info("[conversionVisualChartData] [conversion size: {}]", backUpChartList.size()); + + // query element list. name has been processed + List<Entity> chartElementList = sqlRunner.findAll(this.connection, Entity.create("visual_chart_element")); + Map<String, List<Entity>> elementGroupingByChartIdMap = chartElementList.stream().collect(Collectors.groupingBy(entity -> entity.getStr("chart_id"))); + + // drop alertList type chart + // grouping by panel_id + Map<String, List<Entity>> chartGaroupingByPanelIdMap = backUpChartList.stream() + .filter(entity -> Tool.ObjectUtil.notEqual("alertList", entity.getStr("type"))) + .collect(Collectors.groupingBy(entity -> entity.getStr("panel_id"))); + + for (Map.Entry<String, List<Entity>> entry : chartGaroupingByPanelIdMap.entrySet()) { + String panelId = entry.getKey(); + List<Entity> chartListForPanel = entry.getValue(); + + // handle chart param/datasource + this.handleChartParamFormat(chartListForPanel, elementGroupingByChartIdMap); + + // handle Position chart.x y + this.handlePosition(panelId, chartListForPanel); + + // insert batch + if (Tool.CollUtil.isNotEmpty(chartListForPanel)) { + for (Entity entity : chartListForPanel) { + // reset table name + entity.setTableName("visual_chart"); + } + sqlRunner.insert(this.connection,chartListForPanel); + } + } + log.info("[conversionVisualChartData] [finshed]"); + } + + /** + * 根据下标 获取对应 26 字母,如果超过 26 字母,按照 A-Z AA-AZ BA -BZ AAA-AAZ 顺序排 * - * @param connection + * @param num * @return */ - default SqlConnRunner initSqlRunner(Connection connection) { - log.info("[initSqlRunner] [begin]"); - SqlConnRunner sqlConnRunner = DbUtil.newSqlConnRunner(connection); - log.info("[initSqlRunner] [finshed]"); - return sqlConnRunner; + private String transformNumToLetter(Integer num) { + String letter = ""; + Integer loopNum = num / 26; + if (loopNum > 0) { + letter += transformNumToLetter(loopNum - 1); + } + letter += LETTERS[num % 26]; + return letter; + } + + /** + * 处理位置信息 x/y + * + * @param panelId + * @param chartList + */ + private void handlePosition(String panelId, List<Entity> chartList) { + chartList = chartList.stream().sorted( + Comparator.comparing(entity -> { + String weightStr = Tool.StrUtil.emptyToDefault(entity.getStr("weight"), "0"); + return Integer.valueOf(weightStr); + }) + ).collect(Collectors.toList()); + + if (Tool.StrUtil.equals("0", panelId)) { + // chart template 列表数据不需要处理 只需处理 group 内的 chart ,其他 chart 都设置为 0,0 + Map<String, List<Entity>> chartListGroupByGroupId = chartList.stream().collect(Collectors.groupingBy(entity -> entity.getStr("group_id"))); + for (Map.Entry<String, List<Entity>> entry : chartListGroupByGroupId.entrySet()) { + String groupId = entry.getKey(); + List<Entity> visualChartList = entry.getValue(); + if (Tool.StrUtil.equals("0", groupId)) { + for (Entity visualChart : visualChartList) { + if (Tool.StrUtil.equals(visualChart.getStr("type"), "group")) { + visualChart.set("height", 3); + } + visualChart.set("x", 0); + visualChart.set("y", 0); + } + } else { + // 根据 span and height 计算 XY + this.calculateXYAccordingToSpanAndHeight(visualChartList); + } + } + } else { + // 排序 ,按照 chart - group - chart 顺序排 (group inner chart) 排除在外 + List<Entity> list = new ArrayList<>(chartList.size()); + for (Entity visualChart : chartList) { + if (Tool.StrUtil.equals(visualChart.getStr("type"), "group")) { + // 添加 group + list.add(visualChart); + } else if (!Tool.StrUtil.equals(visualChart.getStr("type"), "group") + && Tool.StrUtil.equals(visualChart.getStr("group_id"), "0")) { + // 非 group & 非 group inner chart + list.add(visualChart); + } + } + // 根据 span and height 计算 XY + this.calculateXYAccordingToSpanAndHeight(list); + + Map<String, List<Entity>> chartListGroupByGroupId = chartList.stream() + .filter(entity -> Tool.ObjectUtil.notEqual("0", entity.getStr("group_id"))) + .collect(Collectors.groupingBy(entity -> entity.getStr("group_id"))); + + for (Map.Entry<String, List<Entity>> entry : chartListGroupByGroupId.entrySet()) { + // 根据 span and height 计算 XY + this.calculateXYAccordingToSpanAndHeight(entry.getValue()); + } + } + } + + /** + * 根据 span and height 计算 XY + * + * @param list + */ + private void calculateXYAccordingToSpanAndHeight(List<Entity> list) { + // 每行 x 计数 + Integer colX = 0; + // 每行 y 计数 + Integer colY = 0; + // 行内最高值 + Integer highestValueInLine = 0; + for (int i = 0; i < list.size(); i++) { + Entity chart = list.get(i); + Integer span = Integer.valueOf(chart.getStr("span")); + // group 默认给 3 就 ok ,前端会根据 组内 chart 自动调整高度 + Integer height = Tool.StrUtil.equals(chart.getStr("type"), "group") ? 3 : Integer.valueOf(chart.getStr("height")); + chart.set("height", height); + colX += span; + // 一行结束 + if (colX > 12) { + colX = span; + // 下一行第一个 Y 就是上一行 height 最大值 + colY += highestValueInLine; + Integer colXValue = colX - span.intValue(); + chart.set("x", colXValue); + chart.set("y", colY); + highestValueInLine = 0; + } else { + // group + if (Tool.StrUtil.equals(chart.getStr("type"), "group")) { + chart.set("x", 0); + } else { + Integer colXValue = colX - span.intValue(); + chart.set("x", colXValue); + } + chart.set("y", colY); + } + // 行内最高值 + highestValueInLine = highestValueInLine < height ? height : highestValueInLine; + } } /** - * 3.0 - 3.2 visual_chart 表数据转换 - * <p> - * 1. 自动添加 chart element name - * 2. 根据图表类型处理 param 内容 - * 3. chart 新增 datasource - * 4. 删除 alertList 类型图表 + * line + * stackArea + * bar + * singleStat + * pie + * table + * alertList + * text + * url + * diagram + * logs + * group * - * @param connection - * @throws Exception + * @param chartListForPanel + * @param elementGroupingByChartIdMap */ - void dataFormatConversion(Connection connection) throws Exception; + private void handleChartParamFormat(List<Entity> chartListForPanel, Map<String, List<Entity>> elementGroupingByChartIdMap) { + Map<String, Object> enableMap = Tool.MapUtil.newHashMap(); + enableMap.put("thresholds", true); + enableMap.put("valueMapping", true); + enableMap.put("legend", true); + + for (Entity visualChart : chartListForPanel) { + try { + // type + String type = visualChart.getStr("type"); + + String paramStr = visualChart.getStr("param"); + paramStr = Tool.StrUtil.emptyToDefault(paramStr, Tool.StrUtil.EMPTY_JSON); + visualChart.set("param", paramStr); + + Map oldParamMap = JSONObject.parseObject(paramStr, Map.class); + String thresholdStrVal = Tool.ObjectUtil.defaultIfNull(oldParamMap.get("threshold"), "").toString(); + String statisticsStrVal = Tool.ObjectUtil.defaultIfNull(oldParamMap.get("statistics"), "").toString(); + + Map<String, Object> params = Tool.MapUtil.newHashMap(); + // thresholds + List<Map> thresholds = new ArrayList<>(); + // valueMapping + List<Map> valueMappingList = new ArrayList<>(); + switch (type) { + // chart-timeSeries line/area + case "line": + case "stackArea": { + // type value 变更 + if (Tool.StrUtil.equalsIgnoreCase(type, "stackArea")) { + visualChart.set("type", "area"); + // area 类型图表 stack 默认值 为 true + params.put("stack", true); + } else { + // line 类型图表 stack 默认值 为 false + params.put("stack", false); + } + List<String> legendOnValueList; + if (oldParamMap.get("legendValue") == null) { + legendOnValueList = Collections.emptyList(); + } else { + Map<String, String> legendValueMap = JSONObject.parseObject(oldParamMap.get("legendValue").toString(), Map.class); + legendOnValueList = legendValueMap.entrySet().stream().filter(entry -> Tool.StrUtil.equals(entry.getValue(), "on")).map(entry -> entry.getKey()).collect(Collectors.toList()); + } + + params.put("style", visualChart.getStr("type")); + // legend + Map legend = Tool.MapUtil.newHashMap(); + legend.put("placement", "bottom"); + legend.put("values", legendOnValueList); + params.put("legend", legend); + + if (Tool.StrUtil.isNotBlank(thresholdStrVal)) { + Map tempThresholdFirstMap = Tool.MapUtil.newHashMap(); + tempThresholdFirstMap.put("color", "#d64f40"); + tempThresholdFirstMap.put("value", thresholdStrVal); + thresholds.add(tempThresholdFirstMap); + } + params.put("thresholds", thresholds); + params.put("nullType", oldParamMap.get("nullType")); + params.put("enable", enableMap); + visualChart.set("datasource", Constant.ChartDatasourceType.METRICS.getValue()); + break; + } + // chart-bar bar + case "bar": { + // none/null 赋默认值 last + params.put("statistics", Tool.StrUtil.equals(statisticsStrVal, "null") || Tool.StrUtil.equals(statisticsStrVal, "") ? "last" : statisticsStrVal); + params.put("nullType", oldParamMap.get("nullType")); + params.put("display", "value"); + params.put("valueMapping", valueMappingList); + List<String> legendOnValueList; + if (oldParamMap.get("legendValue") == null) { + legendOnValueList = Collections.emptyList(); + } else { + Map<String, String> legendValueMap = JSONObject.parseObject(oldParamMap.get("legendValue").toString(), Map.class); + legendOnValueList = legendValueMap.entrySet().stream().filter(entry -> Tool.StrUtil.equals(entry.getValue(), "on")).map(entry -> entry.getKey()).collect(Collectors.toList()); + } + // legend + Map legend = Tool.MapUtil.newHashMap(); + legend.put("placement", "bottom"); + legend.put("values", legendOnValueList); + params.put("legend", legend); + params.put("enable", enableMap); + visualChart.set("datasource", Constant.ChartDatasourceType.METRICS.getValue()); + break; + } + // chart-value stat + case "singleStat": { + visualChart.set("type", "stat"); + // none/null 赋默认值 last + params.put("statistics", Tool.StrUtil.equals(statisticsStrVal, "null") || Tool.StrUtil.equals(statisticsStrVal, "") ? "last" : statisticsStrVal); + params.put("nullType", oldParamMap.get("nullType")); + if (oldParamMap.get("valueMapping") != null) { + Map valueMappingMap = JSONObject.parseObject(oldParamMap.get("valueMapping").toString(), Map.class); + if (Tool.CollectionUtil.isNotEmpty(valueMappingMap)) { + String valueMappingMapType = Tool.ObjectUtil.defaultIfNull(valueMappingMap.get("type"), "").toString(); + List<Map> mappingList = JSONObject.parseArray(valueMappingMap.get("mapping").toString(), Map.class); + for (Map map : mappingList) { + Map tempValueMappingMap = Tool.MapUtil.newHashMap(); + tempValueMappingMap.put("type", valueMappingMapType); + tempValueMappingMap.putAll(map); + valueMappingList.add(tempValueMappingMap); + } + } + } + params.put("valueMapping", valueMappingList); + params.put("enable", enableMap); + visualChart.set("datasource", Constant.ChartDatasourceType.METRICS.getValue()); + break; + } + // chart-pie pie + case "pie": { + // none/null 赋默认值 last + params.put("statistics", Tool.StrUtil.equals(statisticsStrVal, "null") || Tool.StrUtil.equals(statisticsStrVal, "") ? "last" : statisticsStrVal); + params.put("nullType", oldParamMap.get("nullType")); + params.put("display", "value"); + params.put("valueMapping", valueMappingList); + params.put("enable", enableMap); + visualChart.set("datasource", Constant.ChartDatasourceType.METRICS.getValue()); + break; + } + // chart-table table + case "table": { + // none/null 赋默认值 last + params.put("statistics", Tool.StrUtil.equals(statisticsStrVal, "null") || Tool.StrUtil.equals(statisticsStrVal, "") ? "last" : statisticsStrVal); + params.put("nullType", oldParamMap.get("nullType")); + params.put("enable", enableMap); + params.put("indexs", ""); + + List<Entity> visualChartElements = elementGroupingByChartIdMap.get(visualChart.getStr("id")); + String elementDisplay = "", valueDisplay = ""; + for (Entity element : visualChartElements) { + elementDisplay += "{{" + element.getStr("name") + ".$legend}}"; + valueDisplay += "{{" + element.getStr("name") + ".$value}}"; + } + + // columns + List<Map> columns = new ArrayList<>(); + Map elementMap = Tool.MapUtil.newHashMap(); + elementMap.put("title", "Element"); + elementMap.put("show", true); + elementMap.put("unit", 2); + elementMap.put("display", elementDisplay); + columns.add(elementMap); + Map valueMap = Tool.MapUtil.newHashMap(); + valueMap.put("title", "Value"); + valueMap.put("show", true); + valueMap.put("unit", 2); + valueMap.put("display", valueDisplay); + columns.add(valueMap); + + params.put("columns", columns); + + // valueMapping + if (oldParamMap.get("valueMapping") != null) { + Map valueMappingMap = JSONObject.parseObject(oldParamMap.get("valueMapping").toString(), Map.class); + if (Tool.CollectionUtil.isNotEmpty(valueMappingMap)) { + String valueMappingMapType = Tool.ObjectUtil.defaultIfNull(valueMappingMap.get("type"), "").toString(); + List<Map> mappingList = JSONObject.parseArray(valueMappingMap.get("mapping").toString(), Map.class); + for (Map map : mappingList) { + Map tempValueMappingMap = Tool.MapUtil.newHashMap(); + tempValueMappingMap.put("type", valueMappingMapType); + tempValueMappingMap.put("columns", "Value"); + tempValueMappingMap.putAll(map); + valueMappingList.add(tempValueMappingMap); + } + } + } + params.put("valueMapping", valueMappingList); + visualChart.set("datasource", Constant.ChartDatasourceType.METRICS.getValue()); + break; + } + case "group": { + String collapseStr = Tool.ObjectUtil.defaultIfNull(oldParamMap.get("collapse"), "false").toString(); + params.put("collapse", Tool.StrUtil.equals(collapseStr, "true") ? true : false); + visualChart.set("datasource", Constant.ChartDatasourceType.MISC.getValue()); + break; + } + case "text": + case "url": + case "diagram": { + visualChart.set("datasource", Constant.ChartDatasourceType.MISC.getValue()); + break; + } + case "logs": { + visualChart.set("type", "log"); + visualChart.set("datasource", Constant.ChartDatasourceType.LOGS.getValue()); + break; + } + case "assetInfo": + case "endpointInfo": { + visualChart.set("datasource", Constant.ChartDatasourceType.SYSTEM.getValue()); + break; + } + default: { + break; + } + } + // params 处理过 重新赋值 + if (Tool.CollectionUtil.isNotEmpty(params)) { + visualChart.set("param", JSONObject.toJSONString(params)); + } + } catch (Exception e) { + log.error(e, "[handleChartParamFormat] [Data format processing error] [backUp data: {}]", JSONUtil.toJsonStr(visualChart)); + throw new NZException(RCode.ERROR); + } + } + } } diff --git a/nz-admin/src/main/java/com/nis/common/update/v3_0_to_3_2/service/impl/V3_2_DataFormatConversionServiceImpl.java b/nz-admin/src/main/java/com/nis/common/update/v3_0_to_3_2/service/impl/V3_2_DataFormatConversionServiceImpl.java deleted file mode 100644 index a72a5be9..00000000 --- a/nz-admin/src/main/java/com/nis/common/update/v3_0_to_3_2/service/impl/V3_2_DataFormatConversionServiceImpl.java +++ /dev/null @@ -1,525 +0,0 @@ -package com.nis.common.update.v3_0_to_3_2.service.impl; - -import cn.hutool.db.Entity; -import cn.hutool.db.SqlConnRunner; -import cn.hutool.db.handler.EntityListHandler; -import cn.hutool.db.handler.StringHandler; -import cn.hutool.db.sql.SqlExecutor; -import cn.hutool.log.Log; -import com.alibaba.druid.DbType; -import com.alibaba.druid.sql.SQLUtils; -import com.alibaba.fastjson.JSONObject; -import com.nis.common.exception.NZException; -import com.nis.common.update.v3_0_to_3_2.service.V3_2_DataFormatConversionService; -import com.nis.common.utils.Constant; -import com.nis.common.utils.JSONUtil; -import com.nis.common.utils.RCode; -import com.nis.common.utils.Tool; -import org.springframework.stereotype.Service; - -import java.sql.Connection; -import java.sql.SQLException; -import java.util.*; -import java.util.stream.Collectors; -import java.util.stream.Stream; - -/** - * V3_2_DataFormatConversionServiceImpl - * - * @author shizhendong - * @date 2023/03/15 - */ -@Service -public class V3_2_DataFormatConversionServiceImpl implements V3_2_DataFormatConversionService { - - private static final Log log = Log.get(); - - /** - * back up table name - */ - private static final String BACK_UP_TABLE_NAME = "visual_chart_v3_2"; - - private Connection connection; - - private SqlConnRunner sqlRunner; - - private static final String[] LETTERS = new String[]{ - "A", "B", "C", "D", "E", "F", "G", - "H", "I", "J", "K", "L", "M", "N", - "O", "P", "Q", "R", "S", "T", - "U", "V", "W", "X", "Y", "Z"}; - - /** - * before Data Format Conversion - * check backup table is exists - * - * @return - */ - private boolean beforeDataFormatConversion() throws SQLException { - String tableName = SqlExecutor.query(this.connection, "SHOW TABLES LIKE ?", new StringHandler(), BACK_UP_TABLE_NAME); - log.info("[beforeDataFormatConversion] [check the backup table exists] [query result: {}]", tableName); - - if (Tool.StrUtil.isEmpty(tableName)) { - log.warn("[beforeDataFormatConversion] [The backup table does not exist. Do not perform data conversion] [table: {}]", BACK_UP_TABLE_NAME); - return true; - } - - log.info("[beforeDataFormatConversion] [The backup table exists, execute data conversion] [table: {}]", BACK_UP_TABLE_NAME); - return true; - } - - - @Override - public void dataFormatConversion(Connection connection) throws Exception { - // init sql conn runner - this.connection = connection; - sqlRunner = this.initSqlRunner(connection); - - boolean isConversion = this.beforeDataFormatConversion(); - if (!isConversion) { - return; - } - - log.info("[dataFormatConversion] [conversion V3_0 to V3_2 Chart Data]"); - - // handle visual_chart_element name - this.handleElementName(); - - // conversion Visual Chart Data - this.conversionVisualChartData(); - - // remove back up table - log.info("[dataFormatConversion] [remove back up table] [table_name: {}]", BACK_UP_TABLE_NAME); - SqlExecutor.execute(this.connection, Tool.StrUtil.concat(true, "DROP TABLE ", BACK_UP_TABLE_NAME)); - } - - /** - * chart element name 处理 - * 初始化规则: A-Z AA-AZ BA-BZ AAA-AAZ 顺序添加 - */ - private void handleElementName() throws SQLException { - log.info("[handleElementName] [insert chart element name auto] [begin]"); - log.info("[handleElementName] [insert chart element name auto] [rule: {}]", "A-Z AA-AZ BA-BZ AAA-AAZ..."); - - List<Entity> chartElementList = sqlRunner.findAll(this.connection, Entity.create("visual_chart_element")); - Map<Object, List<Entity>> elementGroupingByChartIdMap = chartElementList.stream().collect(Collectors.groupingBy(entity -> entity.get("chart_id"))); - for (Map.Entry<Object, List<Entity>> entry : elementGroupingByChartIdMap.entrySet()) { - List<Entity> elementListForChart = entry.getValue(); - for (int i = 0; i < elementListForChart.size(); i++) { - String letter = this.transformNumToLetter(i); - Entity chartElement = elementListForChart.get(i); - chartElement.set("name", letter); - } - } - List<Entity> allElementList = elementGroupingByChartIdMap.values().stream().flatMap(Collection::stream).collect(Collectors.toList()); - log.info("[handleElementName] [update element size: {}]", allElementList.size()); - if (Tool.CollUtil.isNotEmpty(allElementList)) { - // 防止 sql 过长,分割后拼接 sql 修改 - Integer batchSize = 1000; - Integer limit = (allElementList.size() + batchSize - 1) / batchSize; - List<List<Entity>> splitVisualChartElementList = Stream.iterate(0, n -> n + 1).limit(limit).parallel().map(a -> allElementList.stream().skip(a * batchSize).limit(batchSize).parallel().collect(Collectors.toList())).collect(Collectors.toList()); - - SQLUtils.FormatOption formatOption = new SQLUtils.FormatOption(false, false); - for (List<Entity> batchUpdateElementList : splitVisualChartElementList) { - List<String> list = Tool.ListUtil.list(true); - for (Entity entity : batchUpdateElementList) { - String sql = SQLUtils.format("update visual_chart_element set name = ? where id = ?", DbType.mariadb, Tool.ListUtil.of(entity.getStr("name"), entity.get("id")), formatOption); - list.add(sql); - } - // execute Batch - SqlExecutor.executeBatch(this.connection, list); - } - } - log.info("[handleElementName] [insert chart element name auto] [finshed]"); - } - - /** - * conversion Visual Chart Data - * - * @throws SQLException - */ - private void conversionVisualChartData() throws SQLException { - log.info("[conversionVisualChartData] [begin]"); - - List<Entity> backUpChartList = SqlExecutor.query(this.connection, "SELECT * FROM visual_chart_v3_2 ORDER BY panel_id, group_id, weight", new EntityListHandler()); - log.info("[conversionVisualChartData] [conversion size: {}]", backUpChartList.size()); - - // query element list. name has been processed - List<Entity> chartElementList = sqlRunner.findAll(this.connection, Entity.create("visual_chart_element")); - Map<String, List<Entity>> elementGroupingByChartIdMap = chartElementList.stream().collect(Collectors.groupingBy(entity -> entity.getStr("chart_id"))); - - // drop alertList type chart - // grouping by panel_id - Map<String, List<Entity>> chartGaroupingByPanelIdMap = backUpChartList.stream() - .filter(entity -> Tool.ObjectUtil.notEqual("alertList", entity.getStr("type"))) - .collect(Collectors.groupingBy(entity -> entity.getStr("panel_id"))); - - for (Map.Entry<String, List<Entity>> entry : chartGaroupingByPanelIdMap.entrySet()) { - String panelId = entry.getKey(); - List<Entity> chartListForPanel = entry.getValue(); - - // handle chart param/datasource - this.handleChartParamFormat(chartListForPanel, elementGroupingByChartIdMap); - - // handle Position chart.x y - this.handlePosition(panelId, chartListForPanel); - - // insert batch - if (Tool.CollUtil.isNotEmpty(chartListForPanel)) { - for (Entity entity : chartListForPanel) { - // reset table name - entity.setTableName("visual_chart"); - } - sqlRunner.insert(this.connection,chartListForPanel); - } - } - log.info("[conversionVisualChartData] [finshed]"); - } - - /** - * 根据下标 获取对应 26 字母,如果超过 26 字母,按照 A-Z AA-AZ BA -BZ AAA-AAZ 顺序排 - * - * @param num - * @return - */ - private String transformNumToLetter(Integer num) { - String letter = ""; - Integer loopNum = num / 26; - if (loopNum > 0) { - letter += transformNumToLetter(loopNum - 1); - } - letter += LETTERS[num % 26]; - return letter; - } - - /** - * 处理位置信息 x/y - * - * @param panelId - * @param chartList - */ - private void handlePosition(String panelId, List<Entity> chartList) { - chartList = chartList.stream().sorted( - Comparator.comparing(entity -> { - String weightStr = Tool.StrUtil.emptyToDefault(entity.getStr("weight"), "0"); - return Integer.valueOf(weightStr); - }) - ).collect(Collectors.toList()); - - if (Tool.StrUtil.equals("0", panelId)) { - // chart template 列表数据不需要处理 只需处理 group 内的 chart ,其他 chart 都设置为 0,0 - Map<String, List<Entity>> chartListGroupByGroupId = chartList.stream().collect(Collectors.groupingBy(entity -> entity.getStr("group_id"))); - for (Map.Entry<String, List<Entity>> entry : chartListGroupByGroupId.entrySet()) { - String groupId = entry.getKey(); - List<Entity> visualChartList = entry.getValue(); - if (Tool.StrUtil.equals("0", groupId)) { - for (Entity visualChart : visualChartList) { - if (Tool.StrUtil.equals(visualChart.getStr("type"), "group")) { - visualChart.set("height", 3); - } - visualChart.set("x", 0); - visualChart.set("y", 0); - } - } else { - // 根据 span and height 计算 XY - this.calculateXYAccordingToSpanAndHeight(visualChartList); - } - } - } else { - // 排序 ,按照 chart - group - chart 顺序排 (group inner chart) 排除在外 - List<Entity> list = new ArrayList<>(chartList.size()); - for (Entity visualChart : chartList) { - if (Tool.StrUtil.equals(visualChart.getStr("type"), "group")) { - // 添加 group - list.add(visualChart); - } else if (!Tool.StrUtil.equals(visualChart.getStr("type"), "group") - && Tool.StrUtil.equals(visualChart.getStr("group_id"), "0")) { - // 非 group & 非 group inner chart - list.add(visualChart); - } - } - // 根据 span and height 计算 XY - this.calculateXYAccordingToSpanAndHeight(list); - - Map<String, List<Entity>> chartListGroupByGroupId = chartList.stream() - .filter(entity -> Tool.ObjectUtil.notEqual("0", entity.getStr("group_id"))) - .collect(Collectors.groupingBy(entity -> entity.getStr("group_id"))); - - for (Map.Entry<String, List<Entity>> entry : chartListGroupByGroupId.entrySet()) { - // 根据 span and height 计算 XY - this.calculateXYAccordingToSpanAndHeight(entry.getValue()); - } - } - } - - /** - * 根据 span and height 计算 XY - * - * @param list - */ - private void calculateXYAccordingToSpanAndHeight(List<Entity> list) { - // 每行 x 计数 - Integer colX = 0; - // 每行 y 计数 - Integer colY = 0; - // 行内最高值 - Integer highestValueInLine = 0; - for (int i = 0; i < list.size(); i++) { - Entity chart = list.get(i); - Integer span = Integer.valueOf(chart.getStr("span")); - // group 默认给 3 就 ok ,前端会根据 组内 chart 自动调整高度 - Integer height = Tool.StrUtil.equals(chart.getStr("type"), "group") ? 3 : Integer.valueOf(chart.getStr("height")); - chart.set("height", height); - colX += span; - // 一行结束 - if (colX > 12) { - colX = span; - // 下一行第一个 Y 就是上一行 height 最大值 - colY += highestValueInLine; - Integer colXValue = colX - span.intValue(); - chart.set("x", colXValue); - chart.set("y", colY); - highestValueInLine = 0; - } else { - // group - if (Tool.StrUtil.equals(chart.getStr("type"), "group")) { - chart.set("x", 0); - } else { - Integer colXValue = colX - span.intValue(); - chart.set("x", colXValue); - } - chart.set("y", colY); - } - // 行内最高值 - highestValueInLine = highestValueInLine < height ? height : highestValueInLine; - } - } - - /** - * line - * stackArea - * bar - * singleStat - * pie - * table - * alertList - * text - * url - * diagram - * logs - * group - * - * @param chartListForPanel - * @param elementGroupingByChartIdMap - */ - private void handleChartParamFormat(List<Entity> chartListForPanel, Map<String, List<Entity>> elementGroupingByChartIdMap) { - Map<String, Object> enableMap = Tool.MapUtil.newHashMap(); - enableMap.put("thresholds", true); - enableMap.put("valueMapping", true); - enableMap.put("legend", true); - - for (Entity visualChart : chartListForPanel) { - try { - // type - String type = visualChart.getStr("type"); - - String paramStr = visualChart.getStr("param"); - paramStr = Tool.StrUtil.emptyToDefault(paramStr, Tool.StrUtil.EMPTY_JSON); - visualChart.set("param", paramStr); - - Map oldParamMap = JSONObject.parseObject(paramStr, Map.class); - - String thresholdStrVal = Tool.ObjectUtil.defaultIfNull(oldParamMap.get("threshold"), "").toString(); - String statisticsStrVal = Tool.ObjectUtil.defaultIfNull(oldParamMap.get("statistics"), "").toString(); - - Map<String, Object> params = Tool.MapUtil.newHashMap(); - // thresholds - List<Map> thresholds = new ArrayList<>(); - // valueMapping - List<Map> valueMappingList = new ArrayList<>(); - switch (type) { - // chart-timeSeries line/area - case "line": - case "stackArea": { - // type value 变更 - if (Tool.StrUtil.equalsIgnoreCase(type, "stackArea")) { - visualChart.set("type", "area"); - // area 类型图表 stack 默认值 为 true - params.put("stack", true); - } else { - // line 类型图表 stack 默认值 为 false - params.put("stack", false); - } - List<String> legendOnValueList; - if (oldParamMap.get("legendValue") == null) { - legendOnValueList = Collections.emptyList(); - } else { - Map<String, String> legendValueMap = JSONObject.parseObject(oldParamMap.get("legendValue").toString(), Map.class); - legendOnValueList = legendValueMap.entrySet().stream().filter(entry -> Tool.StrUtil.equals(entry.getValue(), "on")).map(entry -> entry.getKey()).collect(Collectors.toList()); - } - - params.put("style", visualChart.getStr("type")); - // legend - Map legend = Tool.MapUtil.newHashMap(); - legend.put("placement", "bottom"); - legend.put("values", legendOnValueList); - params.put("legend", legend); - - if (Tool.StrUtil.isNotBlank(thresholdStrVal)) { - Map tempThresholdFirstMap = Tool.MapUtil.newHashMap(); - tempThresholdFirstMap.put("color", "#d64f40"); - tempThresholdFirstMap.put("value", thresholdStrVal); - thresholds.add(tempThresholdFirstMap); - } - params.put("thresholds", thresholds); - params.put("nullType", oldParamMap.get("nullType")); - params.put("enable", enableMap); - visualChart.set("datasource", Constant.ChartDatasourceType.METRICS.getValue()); - break; - } - // chart-bar bar - case "bar": { - // none/null 赋默认值 last - params.put("statistics", Tool.StrUtil.equals(statisticsStrVal, "null") || Tool.StrUtil.equals(statisticsStrVal, "") ? "last" : statisticsStrVal); - params.put("nullType", oldParamMap.get("nullType")); - params.put("display", "value"); - params.put("valueMapping", valueMappingList); - List<String> legendOnValueList; - if (oldParamMap.get("legendValue") == null) { - legendOnValueList = Collections.emptyList(); - } else { - Map<String, String> legendValueMap = JSONObject.parseObject(oldParamMap.get("legendValue").toString(), Map.class); - legendOnValueList = legendValueMap.entrySet().stream().filter(entry -> Tool.StrUtil.equals(entry.getValue(), "on")).map(entry -> entry.getKey()).collect(Collectors.toList()); - } - // legend - Map legend = Tool.MapUtil.newHashMap(); - legend.put("placement", "bottom"); - legend.put("values", legendOnValueList); - params.put("legend", legend); - params.put("enable", enableMap); - visualChart.set("datasource", Constant.ChartDatasourceType.METRICS.getValue()); - break; - } - // chart-value stat - case "singleStat": { - visualChart.set("type", "stat"); - // none/null 赋默认值 last - params.put("statistics", Tool.StrUtil.equals(statisticsStrVal, "null") || Tool.StrUtil.equals(statisticsStrVal, "") ? "last" : statisticsStrVal); - params.put("nullType", oldParamMap.get("nullType")); - if (oldParamMap.get("valueMapping") != null) { - Map valueMappingMap = JSONObject.parseObject(oldParamMap.get("valueMapping").toString(), Map.class); - if (Tool.CollectionUtil.isNotEmpty(valueMappingMap)) { - String valueMappingMapType = Tool.ObjectUtil.defaultIfNull(valueMappingMap.get("type"), "").toString(); - List<Map> mappingList = JSONObject.parseArray(valueMappingMap.get("mapping").toString(), Map.class); - for (Map map : mappingList) { - Map tempValueMappingMap = Tool.MapUtil.newHashMap(); - tempValueMappingMap.put("type", valueMappingMapType); - tempValueMappingMap.putAll(map); - valueMappingList.add(tempValueMappingMap); - } - } - } - params.put("valueMapping", valueMappingList); - params.put("enable", enableMap); - visualChart.set("datasource", Constant.ChartDatasourceType.METRICS.getValue()); - break; - } - // chart-pie pie - case "pie": { - // none/null 赋默认值 last - params.put("statistics", Tool.StrUtil.equals(statisticsStrVal, "null") || Tool.StrUtil.equals(statisticsStrVal, "") ? "last" : statisticsStrVal); - params.put("nullType", oldParamMap.get("nullType")); - params.put("display", "value"); - params.put("valueMapping", valueMappingList); - params.put("enable", enableMap); - visualChart.set("datasource", Constant.ChartDatasourceType.METRICS.getValue()); - break; - } - // chart-table table - case "table": { - // none/null 赋默认值 last - params.put("statistics", Tool.StrUtil.equals(statisticsStrVal, "null") || Tool.StrUtil.equals(statisticsStrVal, "") ? "last" : statisticsStrVal); - params.put("nullType", oldParamMap.get("nullType")); - params.put("enable", enableMap); - params.put("indexs", ""); - - List<Entity> visualChartElements = elementGroupingByChartIdMap.get(visualChart.getStr("id")); - String elementDisplay = "", valueDisplay = ""; - for (Entity element : visualChartElements) { - elementDisplay += "{{" + element.getStr("name") + ".$legend}}"; - valueDisplay += "{{" + element.getStr("name") + ".$value}}"; - } - - // columns - List<Map> columns = new ArrayList<>(); - Map elementMap = Tool.MapUtil.newHashMap(); - elementMap.put("title", "Element"); - elementMap.put("show", true); - elementMap.put("unit", 2); - elementMap.put("display", elementDisplay); - columns.add(elementMap); - Map valueMap = Tool.MapUtil.newHashMap(); - valueMap.put("title", "Value"); - valueMap.put("show", true); - valueMap.put("unit", 2); - valueMap.put("display", valueDisplay); - columns.add(valueMap); - - params.put("columns", columns); - - // valueMapping - if (oldParamMap.get("valueMapping") != null) { - Map valueMappingMap = JSONObject.parseObject(oldParamMap.get("valueMapping").toString(), Map.class); - if (Tool.CollectionUtil.isNotEmpty(valueMappingMap)) { - String valueMappingMapType = Tool.ObjectUtil.defaultIfNull(valueMappingMap.get("type"), "").toString(); - List<Map> mappingList = JSONObject.parseArray(valueMappingMap.get("mapping").toString(), Map.class); - for (Map map : mappingList) { - Map tempValueMappingMap = Tool.MapUtil.newHashMap(); - tempValueMappingMap.put("type", valueMappingMapType); - tempValueMappingMap.put("columns", "Value"); - tempValueMappingMap.putAll(map); - valueMappingList.add(tempValueMappingMap); - } - } - } - params.put("valueMapping", valueMappingList); - visualChart.set("datasource", Constant.ChartDatasourceType.METRICS.getValue()); - break; - } - case "group": { - String collapseStr = Tool.ObjectUtil.defaultIfNull(oldParamMap.get("collapse"), "false").toString(); - params.put("collapse", Tool.StrUtil.equals(collapseStr, "true") ? true : false); - visualChart.set("datasource", Constant.ChartDatasourceType.MISC.getValue()); - break; - } - case "text": - case "url": - case "diagram": { - visualChart.set("datasource", Constant.ChartDatasourceType.MISC.getValue()); - break; - } - case "logs": { - visualChart.set("type", "log"); - visualChart.set("datasource", Constant.ChartDatasourceType.LOGS.getValue()); - break; - } - case "assetInfo": - case "endpointInfo": { - visualChart.set("datasource", Constant.ChartDatasourceType.SYSTEM.getValue()); - break; - } - default: { - break; - } - } - // params 处理过 重新赋值 - if (Tool.CollectionUtil.isNotEmpty(params)) { - visualChart.set("param", JSONObject.toJSONString(params)); - } - } catch (Exception e) { - log.error(e, "[handleChartParamFormat] [Data format processing error] [backUp data: {}]", JSONUtil.toJsonStr(visualChart)); - throw new NZException(RCode.ERROR); - } - } - } -} diff --git a/nz-admin/src/main/java/com/nis/common/update/v3_2_to_3_3/service/V3_3_DataFormatConversionService.java b/nz-admin/src/main/java/com/nis/common/update/v3_2_to_3_3/service/V3_3_DataFormatConversionService.java index d726cae3..96b5e002 100644 --- a/nz-admin/src/main/java/com/nis/common/update/v3_2_to_3_3/service/V3_3_DataFormatConversionService.java +++ b/nz-admin/src/main/java/com/nis/common/update/v3_2_to_3_3/service/V3_3_DataFormatConversionService.java @@ -1,42 +1,149 @@ package com.nis.common.update.v3_2_to_3_3.service; -import cn.hutool.db.DbUtil; +import cn.hutool.db.Entity; import cn.hutool.db.SqlConnRunner; +import cn.hutool.db.handler.EntityListHandler; +import cn.hutool.db.handler.StringHandler; +import cn.hutool.db.sql.SqlExecutor; import cn.hutool.log.Log; +import com.nis.common.update.DataFormatConversionInterface; +import com.nis.common.utils.Tool; +import org.springframework.stereotype.Service; import java.sql.Connection; +import java.sql.SQLException; +import java.util.List; +import java.util.Map; +import java.util.function.Function; +import java.util.stream.Collectors; /** - * V3_3_DataFormatConversionService + * V3_3_DataFormatConversionServiceImpl * * @author shizhendong * @date 2023/03/15 */ -public interface V3_3_DataFormatConversionService { +@Service +public class V3_3_DataFormatConversionService implements DataFormatConversionInterface { - Log log = Log.get(); + private static final Log log = Log.get(); /** - * init Sql Runner + * back up table name + */ + private static final String BACK_UP_TABLE_NAME = "asset_model_v3_2"; + + private Connection connection; + + private SqlConnRunner sqlRunner; + + /** + * before Data Format Conversion + * check backup table is exists * - * @param connection * @return */ - default SqlConnRunner initSqlRunner(Connection connection) { - log.info("[initSqlRunner] [begin]"); - SqlConnRunner sqlConnRunner = DbUtil.newSqlConnRunner(connection); - log.info("[initSqlRunner] [finshed]"); - return sqlConnRunner; + private boolean beforeDataFormatConversion() throws SQLException { + String tableName = SqlExecutor.query(this.connection, "SHOW TABLES LIKE ?", new StringHandler(), BACK_UP_TABLE_NAME); + log.info("[beforeDataFormatConversion] [check the backup table exists] [query result: {}]", tableName); + + if (Tool.StrUtil.isEmpty(tableName)) { + log.warn("[beforeDataFormatConversion] [The backup table does not exist. Do not perform data conversion] [table: {}]", BACK_UP_TABLE_NAME); + return false; + } + + log.info("[beforeDataFormatConversion] [The backup table exists, execute data conversion] [table: {}]", BACK_UP_TABLE_NAME); + return true; + } + + @Override + public void dataFormatConversion(Connection connection) throws Exception { + // init sql conn runner + this.connection = connection; + sqlRunner = this.initSqlRunner(connection); + + boolean isConversion = this.beforeDataFormatConversion(); + if (!isConversion) { + return; + } + + log.info("[dataFormatConversion] [conversion V3_2 to V3_3 Asset Model Data]"); + + // conversion Asset Model Data + this.conversionAssetModelData(); + + // remove back up table + log.info("[dataFormatConversion] [remove back up table] [table_name: {}]", BACK_UP_TABLE_NAME); + SqlExecutor.execute(this.connection, Tool.StrUtil.concat(true, "DROP TABLE ", BACK_UP_TABLE_NAME)); } /** - * 1. asset_model 新增 type_id、sys_object_id 列 - * 2. 根据 asset_asset 表关联 model_id & type_id 对应关系,处理 model.typeId + * conversion Asset Model Data * 注:未被 asset 引用的 model 数据直接删除,不再保留 * - * @throws Exception - * @param connection + * @throws SQLException */ - void dataFormatConversion(Connection connection) throws Exception; + private void conversionAssetModelData() throws SQLException { + log.info("[conversionAssetModelData] [begin]"); + + List<Entity> backUpModelList = sqlRunner.findAll(this.connection, Entity.create("asset_model_v3_2")); + Map<String, Entity> backUpModelIdAndEntityMap = backUpModelList.stream().collect(Collectors.toMap(entity -> entity.getStr("id"), Function.identity())); + log.info("[conversionAssetModelData] [conversion size: {}]", backUpModelList.size()); + + List<Entity> assetTypeConfList = sqlRunner.findAll(this.connection, Entity.create("asset_type_conf")); + Map<String, String> assetTypeIdAndNameMap = assetTypeConfList.stream() + .collect(Collectors.toMap(entity -> entity.getStr("id"), entity -> entity.getStr("name"))); + + // 统计 asset_asset 表 model_id 与 type_id 关联关系 + // 注:未被 asset 引用的 model 数据直接删除,不再保留 + List<Entity> modelIdAndTypeIdsMapList = SqlExecutor.query(this.connection, "SELECT model_id, GROUP_CONCAT( DISTINCT type_id ) AS type_ids FROM asset_asset GROUP BY model_id", new EntityListHandler()); + Map<String, String> modelIdAndTypeIdsMap = modelIdAndTypeIdsMapList.stream() + .collect(Collectors.toMap(entity -> entity.getStr("model_id"), entity -> entity.getStr("type_ids"))); + + // 判断 typeids 是否包含 ',' + // 包含',': model 对应多个的 asset_type_conf + // 不包含 ',': model 对应唯一的 asset_type_conf + for (Map.Entry<String, String> entry : modelIdAndTypeIdsMap.entrySet()) { + String modelId = entry.getKey(); + String typeIds = entry.getValue(); + if (Tool.StrUtil.contains(typeIds, ",")) { + // 包含',': model 对应多个的 asset_type_conf + // 新增 model,并设置不同的 type_id,model.name = model.name-type.name + List<String> typeIdStrList = Tool.StrUtil.split(typeIds, ','); + for (String typeIdStr : typeIdStrList) { + Entity entity = backUpModelIdAndEntityMap.get(modelId); + + entity.remove("id"); + entity.set("type_id", Integer.valueOf(typeIdStr)); + entity.set("sys_object_id", Tool.StrUtil.EMPTY); + + // save model.name = oldModelName - typeConfName + String typeConfName = assetTypeIdAndNameMap.get(typeIdStr); + entity.set("name", Tool.StrUtil.concat(true, entity.getStr("name"), "-", typeConfName)); + + // insert model + entity.setTableName("asset_model"); + Long saveEntityId = sqlRunner.insertForGeneratedKey(this.connection, entity); + + // 修改 asset 对应 model_id + // sql: update asset_asset set model_id = {newModelID} where model_id = {oldModelID} and type_id = {typeID} + sqlRunner.update(this.connection, + Entity.create().set("model_id", saveEntityId), + Entity.create("asset_asset").set("model_id", modelId).set("type_id", typeIdStr)); + } + } else { + // 不包含 ',' model 对应唯一的 asset_type_conf, set model.typeId = typeId + Entity entity = backUpModelIdAndEntityMap.get(modelId); + entity.set("type_id", Integer.valueOf(typeIds)); + entity.set("sys_object_id", Tool.StrUtil.EMPTY); + + // insert model + entity.setTableName("asset_model"); + sqlRunner.insert(this.connection, entity); + } + } + log.info("[conversionAssetModelData] [finshed]"); + } } + diff --git a/nz-admin/src/main/java/com/nis/common/update/v3_2_to_3_3/service/impl/V3_3_DataFormatConversionServiceImpl.java b/nz-admin/src/main/java/com/nis/common/update/v3_2_to_3_3/service/impl/V3_3_DataFormatConversionServiceImpl.java deleted file mode 100644 index c2fb864b..00000000 --- a/nz-admin/src/main/java/com/nis/common/update/v3_2_to_3_3/service/impl/V3_3_DataFormatConversionServiceImpl.java +++ /dev/null @@ -1,149 +0,0 @@ -package com.nis.common.update.v3_2_to_3_3.service.impl; - -import cn.hutool.db.Entity; -import cn.hutool.db.SqlConnRunner; -import cn.hutool.db.handler.EntityListHandler; -import cn.hutool.db.handler.StringHandler; -import cn.hutool.db.sql.SqlExecutor; -import cn.hutool.log.Log; -import com.nis.common.update.v3_2_to_3_3.service.V3_3_DataFormatConversionService; -import com.nis.common.utils.Tool; -import org.springframework.stereotype.Service; - -import java.sql.Connection; -import java.sql.SQLException; -import java.util.List; -import java.util.Map; -import java.util.function.Function; -import java.util.stream.Collectors; - -/** - * V3_3_DataFormatConversionServiceImpl - * - * @author shizhendong - * @date 2023/03/15 - */ -@Service -public class V3_3_DataFormatConversionServiceImpl implements V3_3_DataFormatConversionService { - - private static final Log log = Log.get(); - - /** - * back up table name - */ - private static final String BACK_UP_TABLE_NAME = "asset_model_v3_2"; - - private Connection connection; - - private SqlConnRunner sqlRunner; - - /** - * before Data Format Conversion - * check backup table is exists - * - * @return - */ - private boolean beforeDataFormatConversion() throws SQLException { - String tableName = SqlExecutor.query(this.connection, "SHOW TABLES LIKE ?", new StringHandler(), BACK_UP_TABLE_NAME); - log.info("[beforeDataFormatConversion] [check the backup table exists] [query result: {}]", tableName); - - if (Tool.StrUtil.isEmpty(tableName)) { - log.warn("[beforeDataFormatConversion] [The backup table does not exist. Do not perform data conversion] [table: {}]", BACK_UP_TABLE_NAME); - return false; - } - - log.info("[beforeDataFormatConversion] [The backup table exists, execute data conversion] [table: {}]", BACK_UP_TABLE_NAME); - return true; - } - - @Override - public void dataFormatConversion(Connection connection) throws Exception { - // init sql conn runner - this.connection = connection; - sqlRunner = this.initSqlRunner(connection); - - boolean isConversion = this.beforeDataFormatConversion(); - if (!isConversion) { - return; - } - - log.info("[dataFormatConversion] [conversion V3_2 to V3_3 Asset Model Data]"); - - // conversion Asset Model Data - this.conversionAssetModelData(); - - // remove back up table - log.info("[dataFormatConversion] [remove back up table] [table_name: {}]", BACK_UP_TABLE_NAME); - SqlExecutor.execute(this.connection, Tool.StrUtil.concat(true, "DROP TABLE ", BACK_UP_TABLE_NAME)); - } - - /** - * conversion Asset Model Data - * 注:未被 asset 引用的 model 数据直接删除,不再保留 - * - * @throws SQLException - */ - private void conversionAssetModelData() throws SQLException { - log.info("[conversionAssetModelData] [begin]"); - - List<Entity> backUpModelList = sqlRunner.findAll(this.connection, Entity.create("asset_model_v3_2")); - Map<String, Entity> backUpModelIdAndEntityMap = backUpModelList.stream().collect(Collectors.toMap(entity -> entity.getStr("id"), Function.identity())); - log.info("[conversionAssetModelData] [conversion size: {}]", backUpModelList.size()); - - List<Entity> assetTypeConfList = sqlRunner.findAll(this.connection, Entity.create("asset_type_conf")); - Map<String, String> assetTypeIdAndNameMap = assetTypeConfList.stream() - .collect(Collectors.toMap(entity -> entity.getStr("id"), entity -> entity.getStr("name"))); - - // 统计 asset_asset 表 model_id 与 type_id 关联关系 - // 注:未被 asset 引用的 model 数据直接删除,不再保留 - List<Entity> modelIdAndTypeIdsMapList = SqlExecutor.query(this.connection, "SELECT model_id, GROUP_CONCAT( DISTINCT type_id ) AS type_ids FROM asset_asset GROUP BY model_id", new EntityListHandler()); - Map<String, String> modelIdAndTypeIdsMap = modelIdAndTypeIdsMapList.stream() - .collect(Collectors.toMap(entity -> entity.getStr("model_id"), entity -> entity.getStr("type_ids"))); - - // 判断 typeids 是否包含 ',' - // 包含',': model 对应多个的 asset_type_conf - // 不包含 ',': model 对应唯一的 asset_type_conf - for (Map.Entry<String, String> entry : modelIdAndTypeIdsMap.entrySet()) { - String modelId = entry.getKey(); - String typeIds = entry.getValue(); - if (Tool.StrUtil.contains(typeIds, ",")) { - // 包含',': model 对应多个的 asset_type_conf - // 新增 model,并设置不同的 type_id,model.name = model.name-type.name - List<String> typeIdStrList = Tool.StrUtil.split(typeIds, ','); - for (String typeIdStr : typeIdStrList) { - Entity entity = backUpModelIdAndEntityMap.get(modelId); - - entity.remove("id"); - entity.set("type_id", Integer.valueOf(typeIdStr)); - entity.set("sys_object_id", Tool.StrUtil.EMPTY); - - // save model.name = oldModelName - typeConfName - String typeConfName = assetTypeIdAndNameMap.get(typeIdStr); - entity.set("name", Tool.StrUtil.concat(true, entity.getStr("name"), "-", typeConfName)); - - // insert model - entity.setTableName("asset_model"); - Long saveEntityId = sqlRunner.insertForGeneratedKey(this.connection, entity); - - // 修改 asset 对应 model_id - // sql: update asset_asset set model_id = {newModelID} where model_id = {oldModelID} and type_id = {typeID} - sqlRunner.update(this.connection, - Entity.create().set("model_id", saveEntityId), - Entity.create("asset_asset").set("model_id", modelId).set("type_id", typeIdStr)); - } - } else { - // 不包含 ',' model 对应唯一的 asset_type_conf, set model.typeId = typeId - Entity entity = backUpModelIdAndEntityMap.get(modelId); - entity.set("type_id", Integer.valueOf(typeIds)); - entity.set("sys_object_id", Tool.StrUtil.EMPTY); - - // insert model - entity.setTableName("asset_model"); - sqlRunner.insert(this.connection, entity); - } - } - log.info("[conversionAssetModelData] [finshed]"); - } - -} - diff --git a/nz-admin/src/main/java/com/nis/common/update/v3_5_to_3_6/service/V3_6_DataFormatConversionService.java b/nz-admin/src/main/java/com/nis/common/update/v3_5_to_3_6/service/V3_6_DataFormatConversionService.java index 10f3bafb..ed947ded 100644 --- a/nz-admin/src/main/java/com/nis/common/update/v3_5_to_3_6/service/V3_6_DataFormatConversionService.java +++ b/nz-admin/src/main/java/com/nis/common/update/v3_5_to_3_6/service/V3_6_DataFormatConversionService.java @@ -1,40 +1,349 @@ package com.nis.common.update.v3_5_to_3_6.service; -import cn.hutool.db.DbUtil; +import cn.hutool.core.util.ObjectUtil; +import cn.hutool.core.util.StrUtil; +import cn.hutool.db.Entity; import cn.hutool.db.SqlConnRunner; +import cn.hutool.db.handler.EntityListHandler; +import cn.hutool.db.handler.StringHandler; +import cn.hutool.db.sql.SqlExecutor; import cn.hutool.log.Log; +import com.alibaba.druid.DbType; +import com.alibaba.druid.sql.SQLUtils; +import com.alibaba.fastjson.JSONObject; +import com.nis.common.update.DataFormatConversionInterface; +import com.nis.common.utils.Constant; +import com.nis.common.utils.JSONUtil; +import com.nis.common.utils.Tool; +import org.springframework.stereotype.Service; import java.sql.Connection; +import java.sql.SQLException; +import java.util.Date; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; +import java.util.stream.Stream; /** - * V3_6_DataFormatConversionService + * V3_6_DataFormatConversionServiceImpl * * @author shizhendong * @date 2023/03/15 */ -public interface V3_6_DataFormatConversionService { +@Service +public class V3_6_DataFormatConversionService implements DataFormatConversionInterface { - Log log = Log.get(); + private static final Log log = Log.get(); /** - * init Sql Runner + * back up table name + */ + private static final String BACK_UP_TABLE_NAME = "asset_model_v3_6"; + + private Connection connection; + + private SqlConnRunner sqlRunner; + + /** + * before Data Format Conversion + * check backup table is exists * - * @param connection * @return */ - default SqlConnRunner initSqlRunner(Connection connection) { - log.info("[initSqlRunner] [begin]"); - SqlConnRunner sqlConnRunner = DbUtil.newSqlConnRunner(connection); - log.info("[initSqlRunner] [finshed]"); - return sqlConnRunner; + private boolean beforeDataFormatConversion() throws SQLException { + String tableName = SqlExecutor.query(this.connection, "SHOW TABLES LIKE ?", new StringHandler(), BACK_UP_TABLE_NAME); + log.info("[beforeDataFormatConversion] [check the backup table exists] [query result: {}]", tableName); + + if (Tool.StrUtil.isEmpty(tableName)) { + log.warn("[beforeDataFormatConversion] [The backup table does not exist. Do not perform data conversion] [table: {}]", BACK_UP_TABLE_NAME); + return false; + } + + log.info("[beforeDataFormatConversion] [The backup table exists, execute data conversion] [table: {}]", BACK_UP_TABLE_NAME); + return true; + } + + @Override + public void dataFormatConversion(Connection connection) throws Exception { + // init sql conn runner + this.connection = connection; + sqlRunner = this.initSqlRunner(connection); + + boolean isConversion = this.beforeDataFormatConversion(); + if (!isConversion) { + return; + } + + // conversion Asset Model Chart Data + log.info("[dataFormatConversion] [conversion Asset Model Chart Data]"); + this.conversionAssetModelChartData(); + + // remove old chart template + List<Entity> removeChartList = sqlRunner.findAll(this.connection, Entity.create("visual_chart").set("panel_id", 0)); + log.info("[dataFormatConversion] [remove old chart template] [size: {}]", removeChartList.size()); + + List<String> removeChartIdList = removeChartList.stream().map(entity -> entity.getStr("id")).collect(Collectors.toList()); + if (Tool.CollUtil.isNotEmpty(removeChartList)) { + sqlRunner.del(this.connection, Entity.create("visual_chart").set("id", removeChartIdList)); + sqlRunner.del(this.connection, Entity.create("visual_chart_element").set("chart_id", removeChartIdList)); + } + + // remove back up table + log.info("[dataFormatConversion] [remove back up table] [table_name: {}]", BACK_UP_TABLE_NAME); + SqlExecutor.execute(this.connection, Tool.StrUtil.concat(true, "DROP TABLE ", BACK_UP_TABLE_NAME)); + } + + /** + * conversion Asset Model Chart Data + */ + private void conversionAssetModelChartData() throws SQLException { + log.info("[conversionAssetModelChartData] [begin]"); + + // save chart element + List<Entity> saveBatchChartElementList = Tool.ListUtil.list(true); + + List<Entity> backUpModelList = sqlRunner.findAll(this.connection, Entity.create("asset_model_v3_6")); + log.info("[conversionAssetModelChartData] [conversion size: {}]", backUpModelList.size()); + for (Entity backUpAssetModel : backUpModelList) { + // model id + String modelId = backUpAssetModel.getStr("id"); + // model name + String modelName = backUpAssetModel.getStr("name"); + // chart id list + String chartIds = backUpAssetModel.getStr("chart_ids"); + + log.info("[conversionAssetModelChartData] [conversion data begin.] [model id: {}]", modelId); + if (Tool.StrUtil.isEmpty(chartIds)) { + log.warn("[conversionAssetModelChartData] [no config chart. no conversion] [model id: {}]", modelId); + } else { + List<String> chartIdList = Tool.StrUtil.split(Tool.StrUtil.emptyToDefault(chartIds, Tool.StrUtil.EMPTY), Tool.StrUtil.C_COMMA); + log.info("[conversionAssetModelChartData] [asset model chart list] [size: {}] [info: {}]", chartIdList.size(), JSONUtil.toJsonStr(chartIdList)); + + // add panel template + Long insertPanelId = this.addTemplateTypePanel(modelName); + + // model rel chart list + List<Entity> chartList = sqlRunner.findAll(this.connection, Entity.create("visual_chart").set("panel_id", 0).set("id", chartIdList)); + List<Entity> chartListUnderGroup = sqlRunner.findAll(this.connection, Entity.create("visual_chart").set("panel_id", 0).set("group_id", chartIdList)); + chartList.addAll(chartListUnderGroup); + + // distinct + chartList = chartList.stream().distinct().collect(Collectors.toList()); + log.info("[conversionAssetModelChartData] [conversion data] [model: {}] [chart size: {}]", modelName, chartList.size()); + + // assets panel id list + List<String> assetPanelIdListByTheModel = this.getAssetPanelListByTheModel(modelId); + + // group chart + Map<String, String> groupChartIdMapping = Tool.MapUtil.newHashMap(); + List<Entity> groupChartList = chartList.stream().filter(chart -> Tool.StrUtil.equals(Constant.CHART_GROUP_TYPE, chart.getStr("type"))).collect(Collectors.toList()); + for (Entity groupChart : groupChartList) { + Entity tempGroupChart = ObjectUtil.clone(groupChart); + tempGroupChart.remove("id"); + tempGroupChart.set("panel_id", insertPanelId); + + // save group chart + Long insertTempGroupChartId = sqlRunner.insertForGeneratedKey(this.connection, tempGroupChart); + String insertTempGroupChartIdStr = StrUtil.toString(insertTempGroupChartId); + + String groupChartId = groupChart.getStr("id"); + // save mapping + groupChartIdMapping.put(groupChartId, insertTempGroupChartIdStr); + // update chart pid val. only handle asset chart + this.updateAssetChartTemplateLinkValByTheModel(insertTempGroupChartIdStr, assetPanelIdListByTheModel, groupChartId); + } + + // other chart + List<Entity> otherChartList = chartList.stream().filter(chart -> !Tool.StrUtil.equals(Constant.CHART_GROUP_TYPE, chart.getStr("type"))).collect(Collectors.toList()); + for (Entity chart : otherChartList) { + Entity tempChart = ObjectUtil.clone(chart); + + tempChart.remove("id"); + tempChart.set("panel_id", insertPanelId); + // new group id + String groupId = groupChartIdMapping.get(tempChart.getStr("group_id")); + tempChart.set("group_id", StrUtil.emptyToDefault(groupId, "0")); + + // save chart + Long insertTempChartId = sqlRunner.insertForGeneratedKey(this.connection, tempChart); + String insertTempChartIdStr = StrUtil.toString(insertTempChartId); + + List<Entity> elementList = sqlRunner.findAll(this.connection, Entity.create("visual_chart_element").set("chart_id", chart.getStr("id"))); + for (Entity e : elementList) { + e.remove("id"); + e.set("chart_id", insertTempChartIdStr); + } + saveBatchChartElementList.addAll(elementList); + // update chart pid val. only handle asset chart + this.updateAssetChartTemplateLinkValByTheModel(insertTempChartIdStr, assetPanelIdListByTheModel, chart.getStr("id")); + } + // set panel id + backUpAssetModel.set("panel_id", insertPanelId); + // update asset panel temp_id + this.updateAssetPanelTempIdByTheModel(backUpAssetModel, assetPanelIdListByTheModel, insertPanelId); + } + // save model + backUpAssetModel.setTableName("asset_model"); + Long insertBackUpAssetModelId = sqlRunner.insertForGeneratedKey(this.connection, backUpAssetModel); + log.info("[conversionAssetModelChartData] [conversion data finshed.] [insert id: {}] [name: {}]", insertBackUpAssetModelId, backUpAssetModel.getStr("name")); + } + // save chart element + this.saveTheListOfNewChartElement(saveBatchChartElementList); + log.info("[conversionAssetModelChartData] [finshed]"); } /** - * 支持自动将 已有asset model 配置的 chart template 转换为 dashboard template 实现。 + * save Template Type Panel + * name format: 'model.name template' * - * @param connection - * @throws Exception + * @param modelName + * @return */ - void dataFormatConversion(Connection connection) throws Exception; + private Long addTemplateTypePanel(String modelName) throws SQLException { + // panel name + String panelName = Tool.StrUtil.concat(true, modelName, Tool.StrUtil.SPACE, "template"); + List<Entity> panelListByTempName = SqlExecutor.query(this.connection, "SELECT * FROM visual_panel WHERE type = 'template' and name = ?", new EntityListHandler(), panelName); + if (Tool.CollUtil.isNotEmpty(panelListByTempName)) { + // 重复就拼上 4位 随机字符 + panelName = Tool.StrUtil.concat(true, panelName, "-", Tool.RandomUtil.randomString(4)); + } + + Entity panelEntity = Entity.create("visual_panel"); + panelEntity.set("name", panelName); + panelEntity.set("type", "template"); + panelEntity.set("pid", 0); + panelEntity.set("weight", 0); + panelEntity.set("build_in", 0); + + String initParamStr = "{\"chartShare\":\"none\",\"variables\":[],\"report\":{\"schedule\":{\"repeat\":1,\"etime\":\"\",\"stime\":\"\",\"type\":2,\"nums\":[]},\"receivers\":[],\"enable\":false,\"range\":{\"unit\":\"day\",\"interval\":1,\"type\":\"previous\"}}}"; + panelEntity.set("param", initParamStr); + + panelEntity.set("var_type", Constant.ChartVariableType.ASSET.getValue()); + panelEntity.set("tmpl_id", -1); + + panelEntity.set("create_by", -1); + panelEntity.set("uts", System.currentTimeMillis()); + + // save + Long insertPanelId = sqlRunner.insertForGeneratedKey(this.connection, panelEntity); + + // add Asset Info Chart + String assetInfoTemplJsonStr = SqlExecutor.query(this.connection, "SELECT param_value FROM sys_config WHERE param_key = ?", new StringHandler(), Constant.SYSCONFIG_KEY_ASSET_CHART_TPL); + + Entity chartEntity = Entity.create("visual_chart"); + + JSONObject chartTemplate = JSONObject.parseObject(assetInfoTemplJsonStr); + for (Map.Entry<String, Object> entry : chartTemplate.entrySet()) { + chartEntity.set(entry.getKey(), entry.getValue()); + } + chartEntity.set("panel_id", insertPanelId); + chartEntity.set("group_id", 0); + chartEntity.set("weight", 0); + chartEntity.set("datasource", Constant.ChartDatasourceType.SYSTEM.getValue()); + chartEntity.set("update_by", panelEntity.get("create_by")); + chartEntity.set("update_at", new Date()); + + // insert asset info chart + sqlRunner.insert(this.connection, chartEntity); + return insertPanelId; + } + + + /** + * get Asset Panel ID List By The Model + * + * @param modelId + * @return + * @throws SQLException + */ + private List<String> getAssetPanelListByTheModel(String modelId) throws SQLException { + List<Entity> assetList = sqlRunner.findAll(this.connection, Entity.create("asset_asset").set("model_id", modelId)); + if (Tool.CollUtil.isEmpty(assetList)) { + return Tool.ListUtil.empty(); + } + + List<String> assetIdList = assetList.stream().map(entity -> entity.getStr("id")).collect(Collectors.toList()); + + List<Entity> assetPanelList = sqlRunner.findAll(this.connection, Entity.create("visual_panel").set("type", "asset").set("link", assetIdList)); + List<String> assetPanelIdList = assetPanelList.stream().map(entity -> entity.getStr("id")).collect(Collectors.toList()); + return assetPanelIdList; + } + + + /** + * update Asset Chart Template Link Val By The Model + * + * @param newChartPid + * @param assetPanelIdList + * @param oldChartPid + */ + private void updateAssetChartTemplateLinkValByTheModel(String newChartPid, List<String> assetPanelIdList, String oldChartPid) throws SQLException { + if (Tool.CollUtil.isNotEmpty(assetPanelIdList)) { + sqlRunner.update(this.connection, + // record + Entity.create().set("pid", newChartPid), + // where + Entity.create("visual_chart").set("pid", oldChartPid).set("panel_id", assetPanelIdList)); + } + } + + + /** + * update Asset Panel Temp Id By The Model + * + * @param model + * @param assetPanelIdList + * @param templatePanelId + */ + private void updateAssetPanelTempIdByTheModel(Entity model, List<String> assetPanelIdList, Long templatePanelId) throws SQLException { + if (Tool.CollUtil.isEmpty(assetPanelIdList)) { + log.warn("[updateAssetPanelTempIdByTheModel] [no reference asset panel in this model. not update temp_id] [model info: {}]", model.getStr("id")); + } else { + log.info("[updateAssetPanelTempIdByTheModel] [model info: {}] [tempId: {}]", model.getStr("id"), templatePanelId); + sqlRunner.update(this.connection, + Entity.create().set("tmpl_id", templatePanelId), + Entity.create("visual_panel").set("id", assetPanelIdList) + ); + } + } + + /** + * save The List Of New Chart Element + * + * @param chartElementList + */ + private void saveTheListOfNewChartElement(List<Entity> chartElementList) throws SQLException { + log.info("[saveTheListOfNewChartElement] [size: {}]", chartElementList.size()); + if (Tool.CollUtil.isNotEmpty(chartElementList)) { + + // 防止 sql 过长,分割后拼接 sql 修改 + Integer batchSize = 1000; + Integer limit = (chartElementList.size() + batchSize - 1) / batchSize; + List<List<Entity>> splitVisualChartElementList = Stream.iterate(0, n -> n + 1).limit(limit).parallel().map(a -> chartElementList.stream().skip(a * batchSize).limit(batchSize).parallel().collect(Collectors.toList())).collect(Collectors.toList()); + + SQLUtils.FormatOption formatOption = new SQLUtils.FormatOption(false, false); + for (List<Entity> batchUpdateElementList : splitVisualChartElementList) { + List<String> list = Tool.ListUtil.list(true); + for (Entity entity : batchUpdateElementList) { + String sqlStr = "INSERT INTO visual_chart_element(`id`, `chart_id`, `expression`, `type`, `legend`, `build_in`, `seq`, `name`, `state`) VALUES (null, ?, ?, ?, ?, ?, ?, ?, ?)"; + List<Object> mappingValues = Tool.ListUtil.of( + entity.get("chart_id"), + entity.getStr("expression"), + entity.getStr("type"), + entity.getStr("legend"), + entity.get("build_in"), + entity.getStr("seq"), + entity.getStr("name"), + entity.get("state") + ); + String sql = SQLUtils.format(sqlStr, DbType.mariadb, mappingValues, formatOption); + list.add(sql); + } + // execute Batch + SqlExecutor.executeBatch(this.connection, list); + } + } + } } diff --git a/nz-admin/src/main/java/com/nis/common/update/v3_5_to_3_6/service/impl/V3_6_DataFormatConversionServiceImpl.java b/nz-admin/src/main/java/com/nis/common/update/v3_5_to_3_6/service/impl/V3_6_DataFormatConversionServiceImpl.java deleted file mode 100644 index 966d50af..00000000 --- a/nz-admin/src/main/java/com/nis/common/update/v3_5_to_3_6/service/impl/V3_6_DataFormatConversionServiceImpl.java +++ /dev/null @@ -1,349 +0,0 @@ -package com.nis.common.update.v3_5_to_3_6.service.impl; - -import cn.hutool.core.util.ObjectUtil; -import cn.hutool.core.util.StrUtil; -import cn.hutool.db.Entity; -import cn.hutool.db.SqlConnRunner; -import cn.hutool.db.handler.EntityListHandler; -import cn.hutool.db.handler.StringHandler; -import cn.hutool.db.sql.SqlExecutor; -import cn.hutool.log.Log; -import com.alibaba.druid.DbType; -import com.alibaba.druid.sql.SQLUtils; -import com.alibaba.fastjson.JSONObject; -import com.nis.common.update.v3_5_to_3_6.service.V3_6_DataFormatConversionService; -import com.nis.common.utils.Constant; -import com.nis.common.utils.JSONUtil; -import com.nis.common.utils.Tool; -import org.springframework.stereotype.Service; - -import java.sql.Connection; -import java.sql.SQLException; -import java.util.Date; -import java.util.List; -import java.util.Map; -import java.util.stream.Collectors; -import java.util.stream.Stream; - -/** - * V3_6_DataFormatConversionServiceImpl - * - * @author shizhendong - * @date 2023/03/15 - */ -@Service -public class V3_6_DataFormatConversionServiceImpl implements V3_6_DataFormatConversionService { - - private static final Log log = Log.get(); - - /** - * back up table name - */ - private static final String BACK_UP_TABLE_NAME = "asset_model_v3_6"; - - private Connection connection; - - private SqlConnRunner sqlRunner; - - /** - * before Data Format Conversion - * check backup table is exists - * - * @return - */ - private boolean beforeDataFormatConversion() throws SQLException { - String tableName = SqlExecutor.query(this.connection, "SHOW TABLES LIKE ?", new StringHandler(), BACK_UP_TABLE_NAME); - log.info("[beforeDataFormatConversion] [check the backup table exists] [query result: {}]", tableName); - - if (Tool.StrUtil.isEmpty(tableName)) { - log.warn("[beforeDataFormatConversion] [The backup table does not exist. Do not perform data conversion] [table: {}]", BACK_UP_TABLE_NAME); - return false; - } - - log.info("[beforeDataFormatConversion] [The backup table exists, execute data conversion] [table: {}]", BACK_UP_TABLE_NAME); - return true; - } - - @Override - public void dataFormatConversion(Connection connection) throws Exception { - // init sql conn runner - this.connection = connection; - sqlRunner = this.initSqlRunner(connection); - - boolean isConversion = this.beforeDataFormatConversion(); - if (!isConversion) { - return; - } - - // conversion Asset Model Chart Data - log.info("[dataFormatConversion] [conversion Asset Model Chart Data]"); - this.conversionAssetModelChartData(); - - // remove old chart template - List<Entity> removeChartList = sqlRunner.findAll(this.connection, Entity.create("visual_chart").set("panel_id", 0)); - log.info("[dataFormatConversion] [remove old chart template] [size: {}]", removeChartList.size()); - - List<String> removeChartIdList = removeChartList.stream().map(entity -> entity.getStr("id")).collect(Collectors.toList()); - if (Tool.CollUtil.isNotEmpty(removeChartList)) { - sqlRunner.del(this.connection, Entity.create("visual_chart").set("id", removeChartIdList)); - sqlRunner.del(this.connection, Entity.create("visual_chart_element").set("chart_id", removeChartIdList)); - } - - // remove back up table - log.info("[dataFormatConversion] [remove back up table] [table_name: {}]", BACK_UP_TABLE_NAME); - SqlExecutor.execute(this.connection, Tool.StrUtil.concat(true, "DROP TABLE ", BACK_UP_TABLE_NAME)); - } - - /** - * conversion Asset Model Chart Data - */ - private void conversionAssetModelChartData() throws SQLException { - log.info("[conversionAssetModelChartData] [begin]"); - - // save chart element - List<Entity> saveBatchChartElementList = Tool.ListUtil.list(true); - - List<Entity> backUpModelList = sqlRunner.findAll(this.connection, Entity.create("asset_model_v3_6")); - log.info("[conversionAssetModelChartData] [conversion size: {}]", backUpModelList.size()); - for (Entity backUpAssetModel : backUpModelList) { - // model id - String modelId = backUpAssetModel.getStr("id"); - // model name - String modelName = backUpAssetModel.getStr("name"); - // chart id list - String chartIds = backUpAssetModel.getStr("chart_ids"); - - log.info("[conversionAssetModelChartData] [conversion data begin.] [model id: {}]", modelId); - if (Tool.StrUtil.isEmpty(chartIds)) { - log.warn("[conversionAssetModelChartData] [no config chart. no conversion] [model id: {}]", modelId); - } else { - List<String> chartIdList = Tool.StrUtil.split(Tool.StrUtil.emptyToDefault(chartIds, Tool.StrUtil.EMPTY), Tool.StrUtil.C_COMMA); - log.info("[conversionAssetModelChartData] [asset model chart list] [size: {}] [info: {}]", chartIdList.size(), JSONUtil.toJsonStr(chartIdList)); - - // add panel template - Long insertPanelId = this.addTemplateTypePanel(modelName); - - // model rel chart list - List<Entity> chartList = sqlRunner.findAll(this.connection, Entity.create("visual_chart").set("panel_id", 0).set("id", chartIdList)); - List<Entity> chartListUnderGroup = sqlRunner.findAll(this.connection, Entity.create("visual_chart").set("panel_id", 0).set("group_id", chartIdList)); - chartList.addAll(chartListUnderGroup); - - // distinct - chartList = chartList.stream().distinct().collect(Collectors.toList()); - log.info("[conversionAssetModelChartData] [conversion data] [model: {}] [chart size: {}]", modelName, chartList.size()); - - // assets panel id list - List<String> assetPanelIdListByTheModel = this.getAssetPanelListByTheModel(modelId); - - // group chart - Map<String, String> groupChartIdMapping = Tool.MapUtil.newHashMap(); - List<Entity> groupChartList = chartList.stream().filter(chart -> Tool.StrUtil.equals(Constant.CHART_GROUP_TYPE, chart.getStr("type"))).collect(Collectors.toList()); - for (Entity groupChart : groupChartList) { - Entity tempGroupChart = ObjectUtil.clone(groupChart); - tempGroupChart.remove("id"); - tempGroupChart.set("panel_id", insertPanelId); - - // save group chart - Long insertTempGroupChartId = sqlRunner.insertForGeneratedKey(this.connection, tempGroupChart); - String insertTempGroupChartIdStr = StrUtil.toString(insertTempGroupChartId); - - String groupChartId = groupChart.getStr("id"); - // save mapping - groupChartIdMapping.put(groupChartId, insertTempGroupChartIdStr); - // update chart pid val. only handle asset chart - this.updateAssetChartTemplateLinkValByTheModel(insertTempGroupChartIdStr, assetPanelIdListByTheModel, groupChartId); - } - - // other chart - List<Entity> otherChartList = chartList.stream().filter(chart -> !Tool.StrUtil.equals(Constant.CHART_GROUP_TYPE, chart.getStr("type"))).collect(Collectors.toList()); - for (Entity chart : otherChartList) { - Entity tempChart = ObjectUtil.clone(chart); - - tempChart.remove("id"); - tempChart.set("panel_id", insertPanelId); - // new group id - String groupId = groupChartIdMapping.get(tempChart.getStr("group_id")); - tempChart.set("group_id", StrUtil.emptyToDefault(groupId, "0")); - - // save chart - Long insertTempChartId = sqlRunner.insertForGeneratedKey(this.connection, tempChart); - String insertTempChartIdStr = StrUtil.toString(insertTempChartId); - - List<Entity> elementList = sqlRunner.findAll(this.connection, Entity.create("visual_chart_element").set("chart_id", chart.getStr("id"))); - for (Entity e : elementList) { - e.remove("id"); - e.set("chart_id", insertTempChartIdStr); - } - saveBatchChartElementList.addAll(elementList); - // update chart pid val. only handle asset chart - this.updateAssetChartTemplateLinkValByTheModel(insertTempChartIdStr, assetPanelIdListByTheModel, chart.getStr("id")); - } - // set panel id - backUpAssetModel.set("panel_id", insertPanelId); - // update asset panel temp_id - this.updateAssetPanelTempIdByTheModel(backUpAssetModel, assetPanelIdListByTheModel, insertPanelId); - } - // save model - backUpAssetModel.setTableName("asset_model"); - Long insertBackUpAssetModelId = sqlRunner.insertForGeneratedKey(this.connection, backUpAssetModel); - log.info("[conversionAssetModelChartData] [conversion data finshed.] [insert id: {}] [name: {}]", insertBackUpAssetModelId, backUpAssetModel.getStr("name")); - } - // save chart element - this.saveTheListOfNewChartElement(saveBatchChartElementList); - log.info("[conversionAssetModelChartData] [finshed]"); - } - - /** - * save Template Type Panel - * name format: 'model.name template' - * - * @param modelName - * @return - */ - private Long addTemplateTypePanel(String modelName) throws SQLException { - // panel name - String panelName = Tool.StrUtil.concat(true, modelName, Tool.StrUtil.SPACE, "template"); - List<Entity> panelListByTempName = SqlExecutor.query(this.connection, "SELECT * FROM visual_panel WHERE type = 'template' and name = ?", new EntityListHandler(), panelName); - if (Tool.CollUtil.isNotEmpty(panelListByTempName)) { - // 重复就拼上 4位 随机字符 - panelName = Tool.StrUtil.concat(true, panelName, "-", Tool.RandomUtil.randomString(4)); - } - - Entity panelEntity = Entity.create("visual_panel"); - panelEntity.set("name", panelName); - panelEntity.set("type", "template"); - panelEntity.set("pid", 0); - panelEntity.set("weight", 0); - panelEntity.set("build_in", 0); - - String initParamStr = "{\"chartShare\":\"none\",\"variables\":[],\"report\":{\"schedule\":{\"repeat\":1,\"etime\":\"\",\"stime\":\"\",\"type\":2,\"nums\":[]},\"receivers\":[],\"enable\":false,\"range\":{\"unit\":\"day\",\"interval\":1,\"type\":\"previous\"}}}"; - panelEntity.set("param", initParamStr); - - panelEntity.set("var_type", Constant.ChartVariableType.ASSET.getValue()); - panelEntity.set("tmpl_id", -1); - - panelEntity.set("create_by", -1); - panelEntity.set("uts", System.currentTimeMillis()); - - // save - Long insertPanelId = sqlRunner.insertForGeneratedKey(this.connection, panelEntity); - - // add Asset Info Chart - String assetInfoTemplJsonStr = SqlExecutor.query(this.connection, "SELECT param_value FROM sys_config WHERE param_key = ?", new StringHandler(), Constant.SYSCONFIG_KEY_ASSET_CHART_TPL); - - Entity chartEntity = Entity.create("visual_chart"); - - JSONObject chartTemplate = JSONObject.parseObject(assetInfoTemplJsonStr); - for (Map.Entry<String, Object> entry : chartTemplate.entrySet()) { - chartEntity.set(entry.getKey(), entry.getValue()); - } - chartEntity.set("panel_id", insertPanelId); - chartEntity.set("group_id", 0); - chartEntity.set("weight", 0); - chartEntity.set("datasource", Constant.ChartDatasourceType.SYSTEM.getValue()); - chartEntity.set("update_by", panelEntity.get("create_by")); - chartEntity.set("update_at", new Date()); - - // insert asset info chart - sqlRunner.insert(this.connection, chartEntity); - return insertPanelId; - } - - - /** - * get Asset Panel ID List By The Model - * - * @param modelId - * @return - * @throws SQLException - */ - private List<String> getAssetPanelListByTheModel(String modelId) throws SQLException { - List<Entity> assetList = sqlRunner.findAll(this.connection, Entity.create("asset_asset").set("model_id", modelId)); - if (Tool.CollUtil.isEmpty(assetList)) { - return Tool.ListUtil.empty(); - } - - List<String> assetIdList = assetList.stream().map(entity -> entity.getStr("id")).collect(Collectors.toList()); - - List<Entity> assetPanelList = sqlRunner.findAll(this.connection, Entity.create("visual_panel").set("type", "asset").set("link", assetIdList)); - List<String> assetPanelIdList = assetPanelList.stream().map(entity -> entity.getStr("id")).collect(Collectors.toList()); - return assetPanelIdList; - } - - - /** - * update Asset Chart Template Link Val By The Model - * - * @param newChartPid - * @param assetPanelIdList - * @param oldChartPid - */ - private void updateAssetChartTemplateLinkValByTheModel(String newChartPid, List<String> assetPanelIdList, String oldChartPid) throws SQLException { - if (Tool.CollUtil.isNotEmpty(assetPanelIdList)) { - sqlRunner.update(this.connection, - // record - Entity.create().set("pid", newChartPid), - // where - Entity.create("visual_chart").set("pid", oldChartPid).set("panel_id", assetPanelIdList)); - } - } - - - /** - * update Asset Panel Temp Id By The Model - * - * @param model - * @param assetPanelIdList - * @param templatePanelId - */ - private void updateAssetPanelTempIdByTheModel(Entity model, List<String> assetPanelIdList, Long templatePanelId) throws SQLException { - if (Tool.CollUtil.isEmpty(assetPanelIdList)) { - log.warn("[updateAssetPanelTempIdByTheModel] [no reference asset panel in this model. not update temp_id] [model info: {}]", model.getStr("id")); - } else { - log.info("[updateAssetPanelTempIdByTheModel] [model info: {}] [tempId: {}]", model.getStr("id"), templatePanelId); - sqlRunner.update(this.connection, - Entity.create().set("tmpl_id", templatePanelId), - Entity.create("visual_panel").set("id", assetPanelIdList) - ); - } - } - - /** - * save The List Of New Chart Element - * - * @param chartElementList - */ - private void saveTheListOfNewChartElement(List<Entity> chartElementList) throws SQLException { - log.info("[saveTheListOfNewChartElement] [size: {}]", chartElementList.size()); - if (Tool.CollUtil.isNotEmpty(chartElementList)) { - - // 防止 sql 过长,分割后拼接 sql 修改 - Integer batchSize = 1000; - Integer limit = (chartElementList.size() + batchSize - 1) / batchSize; - List<List<Entity>> splitVisualChartElementList = Stream.iterate(0, n -> n + 1).limit(limit).parallel().map(a -> chartElementList.stream().skip(a * batchSize).limit(batchSize).parallel().collect(Collectors.toList())).collect(Collectors.toList()); - - SQLUtils.FormatOption formatOption = new SQLUtils.FormatOption(false, false); - for (List<Entity> batchUpdateElementList : splitVisualChartElementList) { - List<String> list = Tool.ListUtil.list(true); - for (Entity entity : batchUpdateElementList) { - String sqlStr = "INSERT INTO visual_chart_element(`id`, `chart_id`, `expression`, `type`, `legend`, `build_in`, `seq`, `name`, `state`) VALUES (null, ?, ?, ?, ?, ?, ?, ?, ?)"; - List<Object> mappingValues = Tool.ListUtil.of( - entity.get("chart_id"), - entity.getStr("expression"), - entity.getStr("type"), - entity.getStr("legend"), - entity.get("build_in"), - entity.getStr("seq"), - entity.getStr("name"), - entity.get("state") - ); - String sql = SQLUtils.format(sqlStr, DbType.mariadb, mappingValues, formatOption); - list.add(sql); - } - // execute Batch - SqlExecutor.executeBatch(this.connection, list); - } - } - } - -} |
