fix: NEZ-3461 修复断网超过 2h,remote_write too old sample 错误rel-24.01.20

 1. prom_cmd_tmpl per-datacenter agent 添加启动命令 --storage.agent.retention.min-time=1h --storage.agent.retention.max-time=2h
 2. prom_yml_tmpl remote_write  新增 sample_age_limit=2h
 3. prometheus_tsdb_out_of_order_time_window out_of_order_time_window=3h

1 files changed, 9 insertions, 0 deletions

diff --git a/nz-admin/src/main/resources/db/V2024.05.07__update sys_config prom_cmd prom_yml conf.sql b/nz-admin/src/main/resources/db/V2024.05.07__update sys_config prom_cmd prom_yml conf.sql
new file mode 100644
index 00000000..9f220781
--- /dev/null
+++ b/nz-admin/src/main/resources/db/V2024.05.07__update sys_config prom_cmd prom_yml conf.sql
@@ -0,0 +1,9 @@
+/**
+ * 1. prom_cmd_tmpl per-datacenter agent 添加启动命令 --storage.agent.retention.min-time=1h --storage.agent.retention.max-time=2h
+ * 2. prom_yml_tmpl remote_write  新增 sample_age_limit=2h
+ * 3. prometheus_tsdb_out_of_order_time_window out_of_order_time_window=3h
+ */
+
+REPLACE INTO `sys_config`(`id`, `param_key`, `param_value`, `status`, `remark`) VALUES (NULL, 'prom_cmd_tmpl', '<#if currentAgent.type == 1>\r\nOPTION=\"--config.file=\'/opt/nezha/nz-agent/prometheus/prometheus.yml\' --web.listen-address=\'0.0.0.0:10091\' --storage.tsdb.retention.time=<#if systemConfig.metrics_storage_type?trim == \"1\">${systemConfig.metrics_storage_retention?trim}d<#else>${systemConfig.prometheus_default_retention_time_when_s3?trim}d</#if> --storage.tsdb.path=\'/data/prometheusData\' --web.enable-admin-api --enable-feature=\'remote-write-receiver\' --web.enable-lifecycle \"\r\n<#else>\r\nOPTION=\"--config.file=\'/opt/nezha/nz-agent/prometheus/prometheus.yml\' --web.listen-address=\'0.0.0.0:10091\' <#if systemConfig.prometheus_agent_mode?trim == \"1\">--enable-feature=remote-write-receiver,agent --storage.agent.retention.min-time=1h --storage.agent.retention.max-time=2h --storage.agent.wal-truncate-frequency=5m<#else> --enable-feature=\'remote-write-receiver\' --storage.tsdb.path=\'/data/prometheusData\' --storage.tsdb.retention.time=<#if systemConfig.metrics_storage_type?trim == \"1\">${systemConfig.metrics_storage_retention?trim}d<#else>${systemConfig.prometheus_default_retention_time_when_s3?trim}d</#if></#if> --web.enable-admin-api --web.enable-lifecycle \"\r\n</#if>', 1, 'prometheus 启动参数模板');
+REPLACE INTO `sys_config`(`id`, `param_key`, `param_value`, `status`, `remark`) VALUES (NULL, 'prom_yml_tmpl', '{\r\n    \"global\": {\r\n        \"scrape_interval\": \"${scrape_interval}s\",\r\n        \"scrape_timeout\": 	<#if scrape_timeout??> \r\n								\"${scrape_timeout}s\",\r\n						    <#else>\r\n								\"10s\",\r\n							</#if>\r\n        \"evaluation_interval\": \"${scrape_interval}s\"\r\n    },\r\n	\r\n	<#-- alert 配置 -->\r\n	<#if isAlert>\r\n	\"alerting\": {\r\n		\"alertmanagers\": [{\r\n			\"path_prefix\": \"${path_prefix}\",\r\n			\"static_configs\": [{\r\n				\"targets\": [\"${alert_url}\"]\r\n			}]\r\n		}],\r\n		\"alert_relabel_configs\": [{\r\n			\"regex\": \"instance\",\r\n			\"action\": \"labeldrop\"\r\n		}, {\r\n			\"regex\": \"replica\",\r\n			\"action\": \"labeldrop\"\r\n		}]\r\n	},\r\n	\"rule_files\": [\"rule.yml\"],\r\n	</#if>\r\n	\r\n	\r\n	\r\n	<#-- per-datacenter 配置 -->\r\n	<#if !isGlobal>\r\n    \"scrape_configs\": [\r\n	<#list jobList as jobPojo>\r\n			{\r\n				\"job_name\": \"${jobPojo.job_name}\",\r\n				\"scheme\": \"${jobPojo.scheme?default(\"http\")}\",\r\n				\"tls_config\":{\r\n					\"insecure_skip_verify\": true\r\n				},\r\n				\"metrics_path\": <#if jobPojo.metrics_path?default(\"\")?trim?length gt 0>\r\n									\"${jobPojo.metrics_path}\",\r\n								<#else>\r\n									\"/metrics\",\r\n								</#if>\r\n				\"static_configs\":[\r\n					<#list jobPojo.staticConfigList as staticConf >\r\n						{\r\n						  \"targets\": [\r\n								<#list staticConf.targets as targetConf>\r\n										\"${targetConf}\"<#sep>,</#sep>\r\n								</#list>\r\n							]\r\n						<#if staticConf.labels ??&& (staticConf.labels?size >0)>\r\n						   ,\r\n						   \"labels\": {\r\n								<#list staticConf.labels?keys as itemKey>\r\n									\"${itemKey}\":<#if staticConf.labels[itemKey]?default(\"\")?trim?length gt 0>\r\n													\"${staticConf.labels[itemKey]}\"\r\n												  <#else>\"\"</#if><#sep>,</#sep>\r\n								</#list>\r\n							} \r\n						</#if>\r\n						}<#sep>,</#sep>\r\n					</#list>\r\n					],\r\n				\"relabel_configs\": [\r\n					{\r\n						\"regex\": \"job\",\r\n						\"action\": \"labeldrop\"\r\n					}\r\n					<#if !jobPojo.instanceRelabel> \r\n					,\r\n					{\r\n						\"regex\": \"instance\",\r\n						\"action\": \"labeldrop\"\r\n					}\r\n					</#if>\r\n				],\r\n				\"metric_relabel_configs\": [\r\n					{\r\n						\"regex\": \"job\",\r\n						\"action\": \"labeldrop\"\r\n					}\r\n					<#if !jobPojo.instanceRelabel> \r\n					,\r\n					{\r\n						\"regex\": \"instance\",\r\n						\"action\": \"labeldrop\"\r\n					}\r\n					</#if>\r\n				]\r\n			}<#sep>,</#sep>\r\n		</#list>\r\n		<#if jobList??&&(jobList?size>0)>\r\n		,\r\n		</#if>\r\n		<#list endpointList as endpoint>\r\n			{\r\n				\"job_name\": ${endpoint.id?c},\r\n				\"tls_config\":{\r\n					\"insecure_skip_verify\": true\r\n				},\r\n				\"metrics_path\": <#if endpoint.config.metrics_path?default(\"\")?trim?length gt 0>\r\n									\"${endpoint.config.metrics_path}\",\r\n								<#else>\r\n									<#if endpoint.config.protocol == \"http\" || endpoint.config.protocol == \"https\">\r\n										\"/metrics\",\r\n									<#else>\r\n										\"/snmp\",\r\n									</#if>\r\n								</#if>\r\n				\"scrape_interval\": 	<#if endpoint.config.scrape_interval?default(\"\")?trim?length gt 0>\r\n										\"${endpoint.config.scrape_interval?c}s\",\r\n									<#else>\r\n										\"${scrape_interval}s\",\r\n									</#if>\r\n				\"scrape_timeout\": 	<#if endpoint.config.scrape_timeout?default(\"\")?trim?length gt 0>\r\n										\"${endpoint.config.scrape_timeout?c}s\",\r\n									<#else>\r\n										<#if scrape_timeout??> \r\n											\"${scrape_timeout}s\",\r\n										<#else>\r\n											\"10s\",\r\n										</#if>\r\n									</#if>\r\n\r\n\r\n\r\n				<#if endpoint.config.basic_auth??&& (endpoint.config.basic_auth?size >0)>\r\n					<#if endpoint.config.basic_auth.username?default(\"\")?trim?length gt 0 && endpoint.config.basic_auth.password?default(\"\")?trim?length gt 0>\r\n					\"basic_auth\":{\r\n						\"username\": \"${endpoint.config.basic_auth.username}\",\r\n						\"password\": \"${endpoint.config.basic_auth.password}\"\r\n						},\r\n					</#if>\r\n				</#if>\r\n				\r\n				<#if endpoint.config.bearer_token??&&endpoint.config.bearer_token?trim?length gt 0>\r\n					\"authorization\":{\r\n						\"type\":\"Bearer\",\r\n						\"credentials\":\"${endpoint.config.bearer_token}\"\r\n					},\r\n				</#if>\r\n\r\n				\r\n				<#-- params 不为空且类型为 http ，添加 params -->\r\n				<#if endpoint.config.protocol == \"http\" || endpoint.config.protocol == \"https\" >\r\n					\"scheme\": \"${endpoint.config.protocol}\",\r\n					<#if endpoint.config.params?? && (endpoint.config.params?size >0) >\r\n						\"params\":{\r\n							<#list endpoint.config.params?keys as itemKey>\r\n							\"${itemKey}\":[\r\n								<#list endpoint.config.params[itemKey] as pojo>\r\n									\"${pojo}\"\r\n									<#sep>,</#sep>\r\n								</#list>\r\n								]\r\n								<#sep>,</#sep>\r\n							</#list>\r\n						},\r\n					</#if>\r\n				<#elseif endpoint.config.protocol == \"snmp\">\r\n				<#-- 类型是 snmp ，组织 param 数据 ， 原来params弃用，因为 snmp_exporter 不支持其他参数 -->\r\n				   \"params\":{\r\n						\"target\":[ \"${endpoint.config.host}:${endpoint.config.port?c}\" ],\r\n						\"module\":[ \"${endpoint.module.name}\" ]\r\n					},\r\n				</#if>\r\n				\r\n				\"static_configs\": [{\r\n					\"targets\": [<#if endpoint.config.protocol == \"http\" || endpoint.config.protocol == \"https\">\r\n									\"${endpoint.config.host}:${endpoint.config.port?c}\"\r\n								<#else>\r\n									\"127.0.0.1:19116\"\r\n								</#if>\r\n								],\r\n					\"labels\": {\r\n						\"datacenter\": 	\"${endpoint.dc.name}\",\r\n                        \"datacenter_id\":${endpoint.dc.id?c},\r\n						\r\n						\"project\": 		\"${endpoint.project.name}\",\r\n						\"project_id\":   ${endpoint.project.id?c},\r\n						\r\n						\"asset\": 		\"${endpoint.asset.name}\",\r\n						\"asset_id\":     ${endpoint.asset.id?c},\r\n						\r\n						\"module\": 		\"${endpoint.module.name}\",\r\n						\"module_id\":    ${endpoint.module.id?c},\r\n						\r\n						\"endpoint\":     \"${endpoint.name}\",\r\n						\"endpoint_id\":  ${endpoint.id?c},\r\n						\r\n						<#if endpoint.asset.parent??&&(endpoint.asset.parent?size >0)>\r\n							\"parent_asset\"      :  \"${endpoint.asset.parent.name}\",\r\n							\"parent_asset_id\"   :   ${endpoint.asset.parent.id?c},\r\n						</#if>\r\n\r\n						\"nz_agent_id\":  ${endpoint.agent.id?c}\r\n												\r\n										\r\n										<#if endpoint.config.labels??&& (endpoint.config.labels?size >0)>\r\n											,\r\n										</#if>\r\n										<#-- 补充用户自定义 Labels -->\r\n										<#if endpoint.config.labels??&& (endpoint.config.labels?size >0)>\r\n											<#list endpoint.config.labels?keys as key>\r\n												<#if key != \"datacenter\" && key != \"datacenter_id\" && key != \"project\" && key != \"project_id\" && key != \"asset\" && key != \"asset_id\" && key != \"module\" && key != \"module_id\" && key != \"endpoint\" && key != \"endpoint_id\">\r\n													\"${key}\":\"${endpoint.config.labels[key]}\"<#sep>,</#sep>\r\n												</#if>\r\n											 </#list>\r\n										</#if>\r\n					}\r\n				}],\r\n				\"metric_relabel_configs\": [\r\n					{\r\n						\"regex\": \"job\",\r\n						\"action\": \"labeldrop\"\r\n					},\r\n					{\r\n						\"regex\": \"instance\",\r\n						\"action\": \"labeldrop\"\r\n					}\r\n					<#if endpoint.config.relabel_config??&& (endpoint.config.relabel_config?size >0)>\r\n						<#list endpoint.config.relabel_config as relabel>\r\n							,{\r\n								<#if relabel.action?? && (relabel.action == \'labeldrop\' || relabel.action == \'labelkeep\')>\r\n									\"regex\": \"${(relabel.regex)!\'\'}\",\r\n									\"action\": \"${(relabel.action)!\'\'}\"\r\n								</#if>\r\n								<#if relabel.action?? && relabel.action == \'labelmap\'>\r\n									\"regex\": \"${(relabel.regex)!\'\'}\",\r\n									\"action\": \"${(relabel.action)!\'\'}\",\r\n									\"replacement\": \"${(relabel.replacement)!\'\'}\"\r\n								</#if>\r\n								<#if relabel.action?? && (relabel.action == \'drop\' || relabel.action == \'keep\')>\r\n									<#if relabel.source_labels??&& (relabel.source_labels?size >0)>\r\n										\"source_labels\":[\r\n											<#list relabel.source_labels as label>\r\n												\"${label!\'\'}\"<#sep>,</#sep>\r\n											</#list>\r\n										],\r\n									</#if>\r\n									\"regex\": \"${(relabel.regex)!\'\'}\",\r\n									\"action\": \"${(relabel.action)!\'\'}\"\r\n								</#if>\r\n								<#if relabel.action?? && relabel.action == \'replace\'>\r\n									<#if relabel.source_labels??&& (relabel.source_labels?size >0)>\r\n										\"source_labels\":[\r\n											<#list relabel.source_labels as label>\r\n												\"${label!\'\'}\"<#sep>,</#sep>\r\n											</#list>\r\n										],\r\n									</#if>\r\n									\"separator\": \"${((relabel.separator!\'\')?length>0)?string((relabel.separator!\'\'),\";\")}\", \r\n									\"regex\": \"${(relabel.regex)!\'\'}\",\r\n									\"replacement\": \"${(relabel.replacement)!\'\'}\",\r\n									\"action\": \"${(relabel.action)!\'\'}\",\r\n									\"target_label\": \"${(relabel.target_label)!\'\'}\"\r\n								</#if>\r\n							}\r\n						</#list>\r\n					</#if>\r\n				]\r\n			}\r\n			<#sep>,</#sep>\r\n		</#list>\r\n		\r\n		<#-- ping 配置 -->\r\n		\r\n		<#if endpointList??&&(endpointList?size>0)>\r\n		,\r\n		</#if>\r\n		\r\n		<#-- ipmi_ip 配置 -->\r\n		<#list blackboxIpmiJobList as asset>\r\n		{\r\n			\"job_name\": 	\"ipmi_ip_ping_${asset.id?c}\",\r\n			\"tls_config\":{\r\n				\"insecure_skip_verify\": true\r\n			},\r\n			\"metrics_path\": \"/probe\",\r\n			\"scrape_interval\": 	<#if asset.assetPingInterval?default(\"\")?trim?length gt 0>\r\n									\"${asset.assetPingInterval}s\",\r\n								<#else>\r\n									\"${scrape_interval}s\",\r\n								</#if>\r\n			\"scrape_timeout\": 	<#if asset.scrape_timeout?default(\"\")?trim?length gt 0>\r\n									\"${asset.scrape_timeout}s\",\r\n								<#else>\r\n									<#if scrape_timeout??> \r\n										\"${scrape_timeout}s\",\r\n									<#else>\r\n										\"10s\",\r\n									</#if>\r\n								</#if>\r\n            \"params\":	{\r\n				\"target\":[ \"${asset.ipmiIp}\" ],\r\n				\"module\":[ \"ping\" ]\r\n			},\r\n			\"static_configs\": [{\r\n				\"targets\": [\"127.0.0.1:19115\"],\r\n				\"labels\": {\r\n					\"datacenter\": 	\"${asset.dc.name}\",\r\n					\"asset\":        \"${asset.name}\",\r\n					\"host\": 	  	\"${asset.ipmiIp}\",\r\n					\"module\":       \"ping\",\r\n					\"source_ip\":    \"${asset.agent.host}\",\r\n					\"nz_agent_id\":  ${asset.agent.id?c},\r\n					\"type\":         \"ipmi\"\r\n				}\r\n			}],\r\n			\"relabel_configs\": [\r\n				{\r\n					\"regex\": \"job\",\r\n					\"action\": \"labeldrop\"\r\n				},\r\n				{\r\n					\"regex\": \"instance\",\r\n					\"action\": \"labeldrop\"\r\n				}\r\n			],\r\n			\"metric_relabel_configs\": [\r\n			    {\r\n					\"regex\": \"job\",\r\n					\"action\": \"labeldrop\"\r\n				},\r\n				{\r\n					\"regex\": \"instance\",\r\n					\"action\": \"labeldrop\"\r\n				}\r\n			]\r\n		}\r\n		<#sep>,</#sep>\r\n		</#list>\r\n		\r\n		<#if blackboxIpmiJobList??&&(blackboxIpmiJobList?size>0)>\r\n		,\r\n		</#if>\r\n		\r\n		<#list blackboxMgtIpJobList as asset>\r\n		{\r\n			\"job_name\": 	\"asset_ping_${asset.id?c}\",\r\n			\"tls_config\":{\r\n				\"insecure_skip_verify\": true\r\n			},\r\n			\"metrics_path\": \"/probe\",\r\n			\"scrape_interval\": 	<#if asset.assetPingInterval?default(\"\")?trim?length gt 0>\r\n									\"${asset.assetPingInterval}s\",\r\n								<#else>\r\n									\"${scrape_interval}s\",\r\n								</#if>\r\n			\"scrape_timeout\": 	<#if asset.scrape_timeout?default(\"\")?trim?length gt 0>\r\n									\"${asset.scrape_timeout}s\",\r\n								<#else>\r\n									<#if scrape_timeout??> \r\n										\"${scrape_timeout}s\",\r\n									<#else>\r\n										\"10s\",\r\n									</#if>\r\n								</#if>\r\n            \"params\":	{\r\n				\"target\":[ \"${asset.manageIp}\" ],\r\n				\"module\":[ \"ping\" ]\r\n			},\r\n			\"static_configs\": [{\r\n				\"targets\": [\"127.0.0.1:19115\"],\r\n				\"labels\": {\r\n					\"datacenter\": 	\"${asset.dc.name}\",\r\n					\"asset\":        \"${asset.name}\",\r\n					\"host\": 	  	\"${asset.manageIp}\",\r\n					\"module\":       \"ping\",\r\n					\"source_ip\":    \"${asset.agent.host}\",\r\n					\"nz_agent_id\":  ${asset.agent.id?c},\r\n					\"type\":         \"mgt\"\r\n				}\r\n			}],\r\n			\"relabel_configs\": [\r\n				{\r\n					\"regex\": \"job\",\r\n					\"action\": \"labeldrop\"\r\n				},\r\n				{\r\n					\"regex\": \"instance\",\r\n					\"action\": \"labeldrop\"\r\n				}\r\n			],\r\n			\"metric_relabel_configs\": [\r\n			    {\r\n					\"regex\": \"job\",\r\n					\"action\": \"labeldrop\"\r\n				},\r\n				{\r\n					\"regex\": \"instance\",\r\n					\"action\": \"labeldrop\"\r\n				}\r\n			]\r\n		}\r\n		<#sep>,</#sep>\r\n		</#list>\r\n    ]\r\n	<#-- cortex 处理 -->\r\n	,\r\n	\"remote_write\": [\r\n		<#list remoteWriteUrlInfos as remoteConf>\r\n		{\r\n			\"url\": \"${remoteConf.url}\",\r\n			\"authorization\": {\r\n				\"type\": \"Bearer\",\r\n				\"credentials\": \"${remoteConf.token}\"\r\n			},\r\n			\"tls_config\":{\r\n				\"insecure_skip_verify\":true\r\n			},\r\n			\"write_relabel_configs\": [\r\n				{\r\n					\"source_labels\": [\"nz_agent_id\"],\r\n					\"regex\": \"${currentAgent.id?c}\",\r\n					\"action\": \"keep\"\r\n				}\r\n			],\r\n			\"queue_config\":{\r\n				\"capacity\": 5000,\r\n			    \"max_samples_per_send\": 1000,\r\n				\"min_shards\": 5,\r\n				\"max_shards\": 10,\r\n				\"min_backoff\": \"20ms\",\r\n				\"min_backoff\": \"50ms\",\r\n				\"sample_age_limit\": \"2h\"\r\n			}\r\n		}\r\n		<#sep>,</#sep>\r\n		</#list>\r\n	]\r\n	<#else>\r\n	<#-- global 配置 -->\r\n	\"scrape_configs\": [\r\n		<#list jobList as jobPojo>\r\n			{\r\n			   \"job_name\": \"${jobPojo.job_name}\",\r\n			   \"scheme\": \"${jobPojo.scheme?default(\"http\")}\",\r\n			   \"tls_config\":{\r\n					\"insecure_skip_verify\": true\r\n				},\r\n				<#if jobPojo.isFederate> \r\n				\"honor_labels\": ${jobPojo.honor_labels},\r\n				\"params\": {\r\n					\"match[]\": [\r\n						\"{__name__=~\\\".+\\\",nz_agent_id = \\\"${jobPojo.excludeAgentId?c}\\\"}\"\r\n					]\r\n				},\r\n				\"authorization\":{\r\n					\"type\":\"Bearer\",\r\n					\"credentials\":\"${jobPojo.token}\"\r\n				},\r\n				</#if>\r\n				\"metrics_path\": <#if jobPojo.metrics_path?default(\"\")?trim?length gt 0>\r\n									\"${jobPojo.metrics_path}\",\r\n								<#else>\r\n									\"/metrics\",\r\n								</#if>\r\n				\"static_configs\":[\r\n					<#list jobPojo.staticConfigList as staticConf >\r\n						{\r\n						  \"targets\": [\r\n								<#list staticConf.targets as targetConf>\r\n										\"${targetConf}\"<#sep>,</#sep>\r\n								</#list>\r\n							]\r\n						<#if staticConf.labels ??&& (staticConf.labels?size >0)>\r\n						   ,\r\n						   \"labels\": {\r\n								<#list staticConf.labels?keys as itemKey>\r\n									\"${itemKey}\":<#if staticConf.labels[itemKey]?default(\"\")?trim?length gt 0>\r\n													\"${staticConf.labels[itemKey]}\"\r\n												  <#else>\"\"</#if><#sep>,</#sep>\r\n								</#list>\r\n							} \r\n						</#if>\r\n						}<#sep>,</#sep>\r\n					</#list>\r\n					],\r\n				\"relabel_configs\": [\r\n					{\r\n						\"regex\": \"job\",\r\n						\"action\": \"labeldrop\"\r\n					}\r\n					<#if !jobPojo.instanceRelabel> \r\n					,\r\n					{\r\n						\"regex\": \"instance\",\r\n						\"action\": \"labeldrop\"\r\n					}\r\n					</#if>\r\n				],\r\n				\"metric_relabel_configs\": [\r\n					{\r\n						\"regex\": \"job\",\r\n						\"action\": \"labeldrop\"\r\n					}\r\n					<#if !jobPojo.instanceRelabel> \r\n					,\r\n					{\r\n						\"regex\": \"instance\",\r\n						\"action\": \"labeldrop\"\r\n					}\r\n					</#if>\r\n				]\r\n			}<#sep>,</#sep>\r\n		</#list>\r\n		\r\n		<#-- ping 配置 -->\r\n		\r\n		<#if jobList??&&(jobList?size>0)>\r\n		,\r\n		</#if>\r\n		\r\n		<#-- ipmi_ip 配置 -->\r\n		<#list blackboxIpmiJobList as asset>\r\n		{\r\n			\"job_name\": 	\"ipmi_ip_ping_${asset.id?c}\",\r\n			\"tls_config\":{\r\n				\"insecure_skip_verify\": true\r\n			},\r\n			\"metrics_path\": \"/probe\",\r\n			\"scrape_interval\": 	<#if asset.assetPingInterval?default(\"\")?trim?length gt 0>\r\n									\"${asset.assetPingInterval}s\",\r\n								<#else>\r\n									\"${scrape_interval}s\",\r\n								</#if>\r\n			\"scrape_timeout\": 	<#if asset.scrape_timeout?default(\"\")?trim?length gt 0>\r\n									\"${asset.scrape_timeout}s\",\r\n								<#else>\r\n									<#if scrape_timeout??> \r\n										\"${scrape_timeout}s\",\r\n									<#else>\r\n										\"10s\",\r\n									</#if>\r\n								</#if>\r\n            \"params\":	{\r\n				\"target\":[ \"${asset.ipmiIp}\" ],\r\n				\"module\":[ \"ping\" ]\r\n			},\r\n			\"static_configs\": [{\r\n				\"targets\": [\"127.0.0.1:19115\"],\r\n				\"labels\": {\r\n					\"datacenter\": 	\"${asset.dc.name}\",\r\n					\"asset\":        \"${asset.name}\",\r\n					\"host\": 	  	\"${asset.ipmiIp}\",\r\n					\"module\":       \"ping\",\r\n					\"source_ip\":    \"${asset.agent.host}\",\r\n					\"nz_agent_id\":  ${asset.agent.id?c},\r\n					\"type\":         \"ipmi\"\r\n				}\r\n			}],\r\n			\"relabel_configs\": [\r\n				{\r\n					\"regex\": \"job\",\r\n					\"action\": \"labeldrop\"\r\n				},\r\n				{\r\n					\"regex\": \"instance\",\r\n					\"action\": \"labeldrop\"\r\n				}\r\n			],\r\n			\"metric_relabel_configs\": [\r\n			    {\r\n					\"regex\": \"job\",\r\n					\"action\": \"labeldrop\"\r\n				},\r\n				{\r\n					\"regex\": \"instance\",\r\n					\"action\": \"labeldrop\"\r\n				}\r\n			]\r\n		}\r\n		<#sep>,</#sep>\r\n		</#list>\r\n		\r\n		<#if blackboxIpmiJobList??&&(blackboxIpmiJobList?size>0)>\r\n		,\r\n		</#if>\r\n		\r\n		<#-- manage_ip 配置 -->\r\n		<#list blackboxMgtIpJobList as asset>\r\n		{\r\n			\"job_name\": 	\"asset_ping_${asset.id?c}\",\r\n			\"tls_config\":{\r\n				\"insecure_skip_verify\": true\r\n			},\r\n			\"metrics_path\": \"/probe\",\r\n			\"scrape_interval\": 	<#if asset.assetPingInterval?default(\"\")?trim?length gt 0>\r\n									\"${asset.assetPingInterval}s\",\r\n								<#else>\r\n									\"${scrape_interval}s\",\r\n								</#if>\r\n			\"scrape_timeout\": 	<#if asset.scrape_timeout?default(\"\")?trim?length gt 0>\r\n									\"${asset.scrape_timeout}s\",\r\n								<#else>\r\n									<#if scrape_timeout??> \r\n										\"${scrape_timeout}s\",\r\n									<#else>\r\n										\"10s\",\r\n									</#if>\r\n								</#if>\r\n            \"params\":	{\r\n				\"target\":[ \"${asset.manageIp}\" ],\r\n				\"module\":[ \"ping\" ]\r\n			},\r\n			\"static_configs\": [{\r\n				\"targets\": [\"127.0.0.1:19115\"],\r\n				\"labels\": {\r\n					\"datacenter\": 	\"${asset.dc.name}\",\r\n					\"asset\":        \"${asset.name}\",\r\n					\"host\": 	  	\"${asset.manageIp}\",\r\n					\"module\":       \"ping\",\r\n					\"source_ip\":    \"${asset.agent.host}\",\r\n					\"nz_agent_id\":  ${asset.agent.id?c},\r\n					\"type\":         \"mgt\"\r\n				}\r\n			}],\r\n			\"relabel_configs\": [\r\n				{\r\n					\"regex\": \"job\",\r\n					\"action\": \"labeldrop\"\r\n				},\r\n				{\r\n					\"regex\": \"instance\",\r\n					\"action\": \"labeldrop\"\r\n				}\r\n			],\r\n			\"metric_relabel_configs\": [\r\n			    {\r\n					\"regex\": \"job\",\r\n					\"action\": \"labeldrop\"\r\n				},\r\n				{\r\n					\"regex\": \"instance\",\r\n					\"action\": \"labeldrop\"\r\n				}\r\n			]\r\n		}\r\n		<#sep>,</#sep>\r\n		</#list>\r\n		\r\n		<#if blackboxMgtIpJobList??&&(blackboxMgtIpJobList?size>0)>\r\n		,\r\n		</#if>\r\n		\r\n		<#list endpointList as endpoint>\r\n			{\r\n				\"job_name\": ${endpoint.id?c},\r\n				\"tls_config\":{\r\n					\"insecure_skip_verify\": true\r\n				},\r\n				\"metrics_path\": <#if endpoint.config.metrics_path?default(\"\")?trim?length gt 0>\r\n									\"${endpoint.config.metrics_path}\",\r\n								<#else>\r\n									<#if endpoint.config.protocol == \"http\" || endpoint.config.protocol == \"https\">\r\n										\"/metrics\",\r\n									<#else>\r\n										\"/snmp\",\r\n									</#if>\r\n								</#if>\r\n				\"scrape_interval\": 	<#if endpoint.config.scrape_interval?default(\"\")?trim?length gt 0>\r\n										\"${endpoint.config.scrape_interval?c}s\",\r\n									<#else>\r\n										\"${scrape_interval}s\",\r\n									</#if>\r\n				\"scrape_timeout\": 	<#if endpoint.config.scrape_timeout?default(\"\")?trim?length gt 0>\r\n										\"${endpoint.config.scrape_timeout?c}s\",\r\n									<#else>\r\n										<#if scrape_timeout??> \r\n											\"${scrape_timeout}s\",\r\n										<#else>\r\n											\"10s\",\r\n										</#if>\r\n									</#if>\r\n\r\n\r\n\r\n				<#if endpoint.config.basic_auth??&& (endpoint.config.basic_auth?size >0)>\r\n					<#if endpoint.config.basic_auth.username?default(\"\")?trim?length gt 0 && endpoint.config.basic_auth.password?default(\"\")?trim?length gt 0>\r\n					\"basic_auth\":{\r\n						\"username\": \"${endpoint.config.basic_auth.username}\",\r\n						\"password\": \"${endpoint.config.basic_auth.password}\"\r\n						},\r\n					</#if>\r\n				</#if>\r\n				\r\n				\r\n				<#if endpoint.config.bearer_token??&&endpoint.config.bearer_token?trim?length gt 0>\r\n					\"authorization\":{\r\n						\"type\":\"Bearer\",\r\n						\"credentials\":\"${endpoint.config.bearer_token}\"\r\n					},\r\n				</#if>\r\n\r\n				\r\n				<#-- params 不为空且类型为 http ，添加 params -->\r\n				<#if endpoint.config.protocol == \"http\" || endpoint.config.protocol == \"https\" >\r\n					\"scheme\": \"${endpoint.config.protocol}\",\r\n					<#if endpoint.config.params?? && (endpoint.config.params?size >0) >\r\n						\"params\":{\r\n							<#list endpoint.config.params?keys as itemKey>\r\n							\"${itemKey}\":[\r\n								<#list endpoint.config.params[itemKey] as pojo>\r\n									\"${pojo}\"\r\n									<#sep>,</#sep>\r\n								</#list>\r\n								]\r\n								<#sep>,</#sep>\r\n							</#list>\r\n						},\r\n					</#if>\r\n				<#elseif endpoint.config.protocol == \"snmp\">\r\n				<#-- 类型是 snmp ，组织 param 数据 ， 原来params弃用，因为 snmp_exporter 不支持其他参数 -->\r\n				   \"params\":{\r\n						\"target\":[ \"${endpoint.config.host}:${endpoint.config.port?c}\" ],\r\n						\"module\":[ \"${endpoint.module.name}\" ]\r\n					},\r\n				</#if>\r\n				\r\n				\"static_configs\": [{\r\n					\"targets\": [<#if endpoint.config.protocol == \"http\" || endpoint.config.protocol == \"https\">\r\n									\"${endpoint.config.host}:${endpoint.config.port?c}\"\r\n								<#else>\r\n									\"127.0.0.1:19116\"\r\n								</#if>\r\n								],\r\n					\"labels\": {\r\n						\"datacenter\": 	\"${endpoint.dc.name}\",\r\n                        \"datacenter_id\":${endpoint.dc.id?c},\r\n						\r\n						\"project\": 		\"${endpoint.project.name}\",\r\n						\"project_id\":   ${endpoint.project.id?c},\r\n						\r\n						\"asset\": 		\"${endpoint.asset.name}\",\r\n						\"asset_id\":     ${endpoint.asset.id?c},\r\n						\r\n						\"module\": 		\"${endpoint.module.name}\",\r\n						\"module_id\":    ${endpoint.module.id?c},\r\n						\r\n						\"endpoint\":     \"${endpoint.name}\",\r\n						\"endpoint_id\":  ${endpoint.id?c},\r\n						\r\n						<#if endpoint.asset.parent??&&(endpoint.asset.parent?size >0)>\r\n							\"parent_asset\"      :  \"${endpoint.asset.parent.name}\",\r\n							\"parent_asset_id\"   :   ${endpoint.asset.parent.id?c},\r\n						</#if>\r\n						\r\n						\"nz_agent_id\":  ${endpoint.agent.id?c}\r\n												\r\n										\r\n										<#if endpoint.config.labels??&& (endpoint.config.labels?size >0)>\r\n											,\r\n										</#if>\r\n										<#-- 补充用户自定义 Labels -->\r\n										<#if endpoint.config.labels??&& (endpoint.config.labels?size >0)>\r\n											<#list endpoint.config.labels?keys as key>\r\n												<#if key != \"datacenter\" && key != \"datacenter_id\" && key != \"project\" && key != \"project_id\" && key != \"asset\" && key != \"asset_id\" && key != \"module\" && key != \"module_id\" && key != \"endpoint\" && key != \"endpoint_id\">\r\n													\"${key}\":\"${endpoint.config.labels[key]}\"<#sep>,</#sep>\r\n												</#if>\r\n											 </#list>\r\n										</#if>\r\n					}\r\n				}],\r\n				\"metric_relabel_configs\": [\r\n					{\r\n						\"regex\": \"job\",\r\n						\"action\": \"labeldrop\"\r\n					},\r\n					{\r\n						\"regex\": \"instance\",\r\n						\"action\": \"labeldrop\"\r\n					}\r\n					<#if endpoint.config.relabel_config??&& (endpoint.config.relabel_config?size >0)>\r\n						<#list endpoint.config.relabel_config as relabel>\r\n							,{\r\n								<#if relabel.action?? && (relabel.action == \'labeldrop\' || relabel.action == \'labelkeep\')>\r\n									\"regex\": \"${(relabel.regex)!\'\'}\",\r\n									\"action\": \"${(relabel.action)!\'\'}\"\r\n								</#if>\r\n								<#if relabel.action?? && relabel.action == \'labelmap\'>\r\n									\"regex\": \"${(relabel.regex)!\'\'}\",\r\n									\"action\": \"${(relabel.action)!\'\'}\",\r\n									\"replacement\": \"${(relabel.replacement)!\'\'}\"\r\n								</#if>\r\n								<#if relabel.action?? && (relabel.action == \'drop\' || relabel.action == \'keep\')>\r\n									<#if relabel.source_labels??&& (relabel.source_labels?size >0)>\r\n										\"source_labels\":[\r\n											<#list relabel.source_labels as label>\r\n												\"${label!\'\'}\"<#sep>,</#sep>\r\n											</#list>\r\n										],\r\n									</#if>\r\n									\"regex\": \"${(relabel.regex)!\'\'}\",\r\n									\"action\": \"${(relabel.action)!\'\'}\"\r\n								</#if>\r\n								<#if relabel.action?? && relabel.action == \'replace\'>\r\n									<#if relabel.source_labels??&& (relabel.source_labels?size >0)>\r\n										\"source_labels\":[\r\n											<#list relabel.source_labels as label>\r\n												\"${label!\'\'}\"<#sep>,</#sep>\r\n											</#list>\r\n										],\r\n									</#if>\r\n									\"separator\": \"${((relabel.separator!\'\')?length>0)?string((relabel.separator!\'\'),\";\")}\", \r\n									\"regex\": \"${(relabel.regex)!\'\'}\",\r\n									\"replacement\": \"${(relabel.replacement)!\'\'}\",\r\n									\"action\": \"${(relabel.action)!\'\'}\",\r\n									\"target_label\": \"${(relabel.target_label)!\'\'}\"\r\n								</#if>\r\n							}\r\n						</#list>\r\n					</#if>\r\n				]\r\n			}\r\n			<#sep>,</#sep>\r\n		</#list>\r\n	\r\n	]\r\n	\r\n	<#-- cortex 处理 -->\r\n	,\r\n	\"remote_write\": [\r\n		<#list remoteWriteUrlInfos as remoteConf>\r\n		{	\r\n			\"url\": \"${remoteConf.url}\",\r\n			\"authorization\": {\r\n				\"type\": \"Bearer\",\r\n				\"credentials\": \"${remoteConf.token}\"\r\n			},\r\n			\"tls_config\":{\r\n				\"insecure_skip_verify\":true\r\n			},\r\n			\"write_relabel_configs\": [\r\n				{\r\n					\"source_labels\": [\"nz_agent_id\"],\r\n					\"regex\": \"${currentAgent.id?c}\",\r\n					\"action\": \"keep\"\r\n				}\r\n			],\r\n			\"queue_config\":{\r\n				\"capacity\": 5000,\r\n			    \"max_samples_per_send\": 1000,\r\n				\"min_shards\": 5,\r\n				\"max_shards\": 10,\r\n				\"min_backoff\": \"20ms\",\r\n				\"min_backoff\": \"50ms\",\r\n				\"sample_age_limit\": \"2h\"\r\n			}\r\n		}\r\n		<#sep>,</#sep>\r\n		</#list>\r\n	]	\r\n	</#if>\r\n	\r\n	<#-- tsdb 配置 -->\r\n	,\r\n	\"storage\": {\r\n		\"tsdb\": {\r\n			\"out_of_order_time_window\": \"${systemConfig.prometheus_tsdb_out_of_order_time_window}h\"\r\n		}\r\n	}\r\n}', 1, 'prometheus config文件模板');
+REPLACE INTO `sys_config`(`id`, `param_key`, `param_value`, `status`, `remark`) VALUES (NULL, 'prometheus_tsdb_out_of_order_time_window', '3', 1, 'prometheus 允许摄取无序样本的最大时长,默认：2h，单位：h');