一.方案选型
对于应用的监控，可以使用actuator进行侵入式指标采集，也可以基于prometheus的jmx_exporter进行非侵入式采集。
使用actuator可以非常方便的进行服务监控端点基于注册中心进行自动发现，但是侵入式本身对于项目来说具有一定的研发改动成本不利于推进，同时在安全上也需要做一些合规措施，因此这个方案推进难度还是较大的。
这里可以采用第二种方法，基于prometheus的jmx_exporter进行非侵入式采集。推进容易,属于推荐方案。
二.收集指标维度
请求 | 应用进出口流量 |
qps | |
平均请求时间 | |
最慢请求时间 | |
内存 | jvm堆内存使用 |
非堆内存使用 | |
老年代使用情况 | |
新生代使用情况 | |
线程 | 线程死锁数 |
线程状态情况 | |
gc | gc次数 |
gc时间 | |
nmt | 内存泄漏监测 |
概况 | 应用存活 |
class加载数 |
基于以上指标数，可以对应用进行较为全面的监控。
三.操作
1.研发启动tomcat相关bean的收集:(tomcat相关监控指标，比如qps等统计需要)
研发在应用的application.yaml文件中修改：
打开相关参数
2.修改应用启动脚本
示例如下：
java -XX:NativeMemoryTracking=summary -javaagent:/xx/jmx_prometheus_javaagent-0.16.1.jar=19013:/xx/config.yaml -XX:+UnlockDiagnosticVMOptions -XX:+PrintNMTStatistics
config.yaml内容如下：
---
lowercaseOutputLabelNames: true
lowercaseOutputName: true
whitelistObjectNames: ["java.lang:type=OperatingSystem"]
blacklistObjectNames: []
rules:
- pattern: 'java.lang<type=OperatingSystem><>(committed_virtual_memory|free_physical_memory|free_swap_space|total_physical_memory|total_swap_space)_size:'
name: os_$1_bytes
type: GAUGE
attrNameSnakeCase: true
- pattern: 'java.lang<type=OperatingSystem><>((?!process_cpu_time)\w+):'
name: os_$1
type: GAUGE
attrNameSnakeCase: true
- pattern: 'Tomcat<type=GlobalRequestProcessor, name=\"(\w+-\w+)-(\d+)\"><>(\w+):'
name: tomcat_$3_total
labels:
port: "$2"
protocol: "$1"
help: Tomcat global $3
type: COUNTER
- pattern: 'Tomcat<j2eeType=Servlet, WebModule=//([-a-zA-Z0-9+&@#/%?=~_|!:.,;]*[-a-zA-Z0-9+&@#/%=~_|]), name=([-a-zA-Z0-9+/$%~_-|!.]*), J2EEApplication=none, J2EEServer=none><>(requestCount|maxTime|processingTime|errorCount):'
name: tomcat_servlet_$3_total
labels:
module: "$1"
servlet: "$2"
help: Tomcat servlet $3 total
type: COUNTER
- pattern: 'Tomcat<type=ThreadPool, name="(\w+-\w+)-(\d+)"><>(currentThreadCount|currentThreadsBusy|keepAliveCount|pollerThreadCount|connectionCount):'
name: tomcat_threadpool_$3
labels:
port: "$2"
protocol: "$1"
help: Tomcat threadpool $3
type: GAUGE
- pattern: 'Tomcat<type=Manager, host=([-a-zA-Z0-9+&@#/%?=~_|!:.,;]*[-a-zA-Z0-9+&@#/%=~_|]), context=([-a-zA-Z0-9+/$%~_-|!.]*)><>(processingTime|sessionCounter|rejectedSessions|expiredSessions):'
name: tomcat_session_$3_total
labels:
context: "$2"
host: "$1"
help: Tomcat session $3 total
type: COUNTER
3.修改完毕后将jmx端点注册到prometheus,示例：
- job_name: "jmx"
static_configs:
- targets: ['ip:port']
启动应用，重载prometheus后即可观察到相关数据。
应用大屏可参考如下：
{
"annotations": {
"list": [
{
"builtIn": 1,
"datasource": "-- Grafana --",
"enable": true,
"hide": true,
"iconColor": "rgba(0, 211, 255, 1)",
"name": "Annotations & Alerts",
"target": {
"limit": 100,
"matchAny": false,
"tags": [],
"type": "dashboard"
},
"type": "dashboard"
}
]
},
"description": "Complete dashboard using metrics from prometheus JMX exporter, with drill down per job > instance",
"editable": true,
"gnetId": 8563,
"graphTooltip": 0,
"id": 45,
"iteration": 1645770479707,
"links": [],
"panels": [
{
"collapsed": false,
"datasource": null,
"gridPos": {
"h": 1,
"w": 24,
"x": 0,
"y": 0
},
"id": 119,
"panels": [],
"title": "应用概况",
"type": "row"
},
{
"cacheTimeout": null,
"datasource": "$datasource",
"fieldConfig": {
"defaults": {
"color": {
"mode": "thresholds"
},
"mappings": [
{
"options": {
"0": {
"text": "DOWN"
},
"1": {
"text": "UP"
}
},
"type": "value"
},
{
"options": {
"match": "null",
"result": {
"text": "DOWN"
}
},
"type": "special"
}
],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "#d44a3a",
"value": null
},
{
"color": "#e24d42",
"value": 0
},
{
"color": "#299c46",
"value": 1
}
]
},
"unit": "none"
},
"overrides": []
},
"gridPos": {
"h": 4,
"w": 4,
"x": 0,
"y": 1
},
"hideTimeOverride": false,
"id": 21,
"interval": null,
"links": [
{
"targetBlank": true,
"title": "Tomcat dashboard",
"url": "/d/chanjarster-tomcat-dashboard/tomcat-dashboard?$__url_time_range&$__all_variables"
}
],
"maxDataPoints": 100,
"options": {
"colorMode": "value",
"graphMode": "none",
"justifyMode": "auto",
"orientation": "horizontal",
"reduceOptions": {
"calcs": [
"lastNotNull"
],
"fields": "",
"values": false
},
"text": {},
"textMode": "auto"
},
"pluginVersion": "8.1.5",
"targets": [
{
"exemplar": true,
"expr": "up{job=\"$job\",instance=\"$instance\",servicetype=\"$service\"}",
"format": "time_series",
"instant": true,
"interval": "",
"intervalFactor": 1,
"legendFormat": "",
"refId": "A"
}
],
"timeShift": null,
"title": "Status",
"type": "stat"
},
{
"cacheTimeout": null,
"datasource": "$datasource",
"fieldConfig": {
"defaults": {
"color": {
"mode": "thresholds"
},
"decimals": 0,
"mappings": [
{
"options": {
"match": "null",
"result": {
"text": "N/A"
}
},
"type": "special"
}
],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
},
{
"color": "red",
"value": 80
}
]
},
"unit": "s"
},
"overrides": []
},
"gridPos": {
"h": 4,
"w": 5,
"x": 4,
"y": 1
},
"id": 14,
"interval": null,
"links": [],
"maxDataPoints": 100,
"options": {
"colorMode": "none",
"graphMode": "none",
"justifyMode": "auto",
"orientation": "horizontal",
"reduceOptions": {
"calcs": [
"lastNotNull"
],
"fields": "",
"values": false
},
"text": {},
"textMode": "auto"
},
"pluginVersion": "8.1.5",
"targets": [
{
"exemplar": true,
"expr": "time() - process_start_time_seconds{job=\"$job\",instance=\"$instance\",servicetype=\"$service\"}",
"format": "time_series",
"instant": true,
"interval": "",
"intervalFactor": 1,
"legendFormat": "",
"refId": "A"
}
],
"title": "Uptime",
"type": "stat"
},
{
"cacheTimeout": null,
"datasource": "$datasource",
"fieldConfig": {
"defaults": {
"color": {
"mode": "thresholds"
},
"mappings": [
{
"options": {
"match": "null",
"result": {
"text": "N/A"
}
},
"type": "special"
}
],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "#299c46",
"value": null
},
{
"color": "rgba(237, 129, 40, 0.89)",
"value": 35
},
{
"color": "#d44a3a",
"value": 50
}
]
},
"unit": "dateTimeAsIso"
},
"overrides": []
},
"gridPos": {
"h": 4,
"w": 5,
"x": 9,
"y": 1
},
"id": 15,
"interval": "",
"links": [],
"maxDataPoints": 100,
"options": {
"colorMode": "none",
"graphMode": "none",
"justifyMode": "auto",
"orientation": "horizontal",
"reduceOptions": {
"calcs": [
"lastNotNull"
],
"fields": "",
"values": false
},
"text": {},
"textMode": "auto"
},
"pluginVersion": "8.1.5",
"targets": [
{
"exemplar": true,
"expr": "process_start_time_seconds{job=\"$job\",instance=\"$instance\",servicetype=\"$service\"}*1000",
"format": "time_series",
"instant": true,
"interval": "",
"intervalFactor": 1,
"legendFormat": "",
"refId": "A"
}
],
"title": "Start time",
"type": "stat"
},
{
"datasource": null,
"fieldConfig": {
"defaults": {
"color": {
"fixedColor": "blue",
"mode": "fixed"
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
},
{
"color": "red",
"value": 80
}
]
}
},
"overrides": []
},
"gridPos": {
"h": 4,
"w": 5,
"x": 14,
"y": 1
},
"id": 103,
"options": {
"colorMode": "value",
"graphMode": "area",
"justifyMode": "auto",
"orientation": "auto",
"reduceOptions": {
"calcs": [
"lastNotNull"
],
"fields": "",
"values": false
},
"text": {},
"textMode": "auto"
},
"pluginVersion": "8.1.5",
"targets": [
{
"exemplar": true,
"expr": "((jvm_memory_bytes_used{agenttype=\"jmx-agent\", area=\"heap\", instance=\"$instance\", job=\"jmx\", servicetype=\"$service\"} / 1024 / 1024 ) / (jvm_memory_bytes_max{agenttype=\"jmx-agent\", area=\"heap\", instance=\"$instance\", job=\"jmx\", servicetype=\"$service\"} / 1024 /1024)) * 100",
"interval": "",
"legendFormat": "",
"refId": "A"
}
],
"title": "堆内存使用百分比",
"type": "stat"
},
{
"datasource": null,
"fieldConfig": {
"defaults": {
"color": {
"fixedColor": "blue",
"mode": "fixed"
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
},
{
"color": "red",
"value": 80
}
]
}
},
"overrides": []
},
"gridPos": {
"h": 4,
"w": 5,
"x": 19,
"y": 1
},
"id": 104,
"options": {
"colorMode": "value",
"graphMode": "area",
"justifyMode": "auto",
"orientation": "auto",
"reduceOptions": {
"calcs": [
"lastNotNull"
],
"fields": "",
"values": false
},
"text": {},
"textMode": "auto"
},
"pluginVersion": "8.1.5",
"targets": [
{
"exemplar": true,
"expr": "((jvm_memory_bytes_used{agenttype=\"jmx-agent\", area=\"nonheap\", instance=\"$instance\", job=\"jmx\", servicetype=\"$service\"} / 1024 / 1024 ) / (jvm_memory_bytes_max{agenttype=\"jmx-agent\", area=\"nonheap\", instance=\"$instance\", job=\"jmx\", servicetype=\"$service\"} / 1024 /1024)) * 100",
"interval": "",
"legendFormat": "",
"refId": "A"
}
],
"title": "非堆内存使用百分比",
"type": "stat"
},
{
"datasource": null,
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"axisLabel": "",
"axisPlacement": "auto",
"barAlignment": 0,
"drawStyle": "line",
"fillOpacity": 0,
"gradientMode": "none",
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
},
"lineInterpolation": "linear",
"lineWidth": 1,
"pointSize": 5,
"scaleDistribution": {
"type": "linear"
},
"showPoints": "never",
"spanNulls": false,
"stacking": {
"group": "A",
"mode": "none"
},
"thresholdsStyle": {
"mode": "off"
}
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
},
{
"color": "red",
"value": 80
}
]
}
},
"overrides": []
},
"gridPos": {
"h": 8,
"w": 12,
"x": 0,
"y": 5
},
"id": 101,
"options": {
"legend": {
"calcs": [
"lastNotNull",
"max"
],
"displayMode": "table",
"placement": "bottom"
},
"tooltip": {
"mode": "multi"
}
},
"targets": [
{
"exemplar": true,
"expr": "process_open_fds{agenttype=\"jmx-agent\", instance=\"$instance\", job=\"jmx\", servicetype=\"$service\"}",
"interval": "",
"legendFormat": "当前打开数",
"refId": "A"
},
{
"exemplar": true,
"expr": "process_max_fds{agenttype=\"jmx-agent\", instance=\"$instance\", job=\"jmx\", servicetype=\"$service\"}",
"hide": false,
"interval": "",
"legendFormat": "最大打开数",
"refId": "B"
}
],
"title": "应用文件打开数",
"type": "timeseries"
},
{
"datasource": null,
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"axisLabel": "",
"axisPlacement": "auto",
"barAlignment": 0,
"drawStyle": "line",
"fillOpacity": 0,
"gradientMode": "none",
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
},
"lineInterpolation": "linear",
"lineWidth": 1,
"pointSize": 5,
"scaleDistribution": {
"type": "linear"
},
"showPoints": "never",
"spanNulls": false,
"stacking": {
"group": "A",
"mode": "none"
},
"thresholdsStyle": {
"mode": "off"
}
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
},
{
"color": "red",
"value": 80
}
]
}
},
"overrides": []
},
"gridPos": {
"h": 8,
"w": 12,
"x": 12,
"y": 5
},
"id": 111,
"options": {
"legend": {
"calcs": [
"lastNotNull",
"max",
"min"
],
"displayMode": "table",
"placement": "bottom"
},
"tooltip": {
"mode": "multi"
}
},
"targets": [
{
"exemplar": true,
"expr": "jvm_classes_loaded{job=\"$job\", instance=\"$instance\"}",
"interval": "",
"legendFormat": "当前打开数",
"refId": "A"
}
],
"title": "class装载数",
"type": "timeseries"
},
{
"collapsed": false,
"datasource": null,
"gridPos": {
"h": 1,
"w": 24,
"x": 0,
"y": 13
},
"id": 84,
"panels": [],
"title": "请求信息",
"type": "row"
},
{
"datasource": null,
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"axisLabel": "",
"axisPlacement": "auto",
"barAlignment": 0,
"drawStyle": "line",
"fillOpacity": 0,
"gradientMode": "none",
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
},
"lineInterpolation": "linear",
"lineWidth": 1,
"pointSize": 5,
"scaleDistribution": {
"type": "linear"
},
"showPoints": "never",
"spanNulls": false,
"stacking": {
"group": "A",
"mode": "none"
},
"thresholdsStyle": {
"mode": "off"
}
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
},
{
"color": "red",
"value": 80
}
]
}
},
"overrides": []
},
"gridPos": {
"h": 8,
"w": 10,
"x": 0,
"y": 14
},
"id": 127,
"options": {
"legend": {
"calcs": [
"last",
"min",
"max"
],
"displayMode": "table",
"placement": "bottom"
},
"tooltip": {
"mode": "multi"
}
},
"targets": [
{
"exemplar": true,
"expr": "increase(tomcat_bytesreceived_total{agenttype=\"jmx-agent\", instance=\"$instance\", job=\"jmx\", servicetype=\"$service\"}[1m]) ",
"interval": "",
"legendFormat": "recv",
"refId": "A"
},
{
"exemplar": true,
"expr": "increase(tomcat_bytessent_total{agenttype=\"jmx-agent\", instance=\"$instance\", job=\"jmx\", servicetype=\"$service\"}[1m])",
"hide": false,
"interval": "",
"legendFormat": "send",
"refId": "B"
}
],
"title": "应用网络流量(Bytes)",
"type": "timeseries"
},
{
"datasource": null,
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"axisLabel": "",
"axisPlacement": "auto",
"barAlignment": 0,
"drawStyle": "line",
"fillOpacity": 0,
"gradientMode": "none",
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
},
"lineInterpolation": "linear",
"lineWidth": 1,
"pointSize": 5,
"scaleDistribution": {
"type": "linear"
},
"showPoints": "never",
"spanNulls": false,
"stacking": {
"group": "A",
"mode": "none"
},
"thresholdsStyle": {
"mode": "off"
}
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
},
{
"color": "red",
"value": 80
}
]
}
},
"overrides": []
},
"gridPos": {
"h": 8,
"w": 9,
"x": 10,
"y": 14
},
"id": 128,
"options": {
"legend": {
"calcs": [
"last",
"min",
"max"
],
"displayMode": "table",
"placement": "bottom"
},
"tooltip": {
"mode": "multi"
}
},
"targets": [
{
"exemplar": true,
"expr": "increase(tomcat_requestcount_total{agenttype=\"jmx-agent\", instance=\"$instance\", job=\"jmx\", servicetype=\"$service\"}[1m]) ",
"interval": "",
"legendFormat": "总请求数",
"refId": "A"
},
{
"exemplar": true,
"expr": "increase(tomcat_errorcount_total{agenttype=\"jmx-agent\", instance=\"$instance\", job=\"jmx\", servicetype=\"$service\"}[1m])",
"hide": false,
"interval": "",
"legendFormat": "error请求数",
"refId": "B"
}
],
"title": "请求数",
"type": "timeseries"
},
{
"datasource": null,
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"axisLabel": "",
"axisPlacement": "auto",
"barAlignment": 0,
"drawStyle": "line",
"fillOpacity": 0,
"gradientMode": "none",
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
},
"lineInterpolation": "linear",
"lineWidth": 1,
"pointSize": 5,
"scaleDistribution": {
"type": "linear"
},
"showPoints": "never",
"spanNulls": false,
"stacking": {
"group": "A",
"mode": "none"
},
"thresholdsStyle": {
"mode": "off"
}
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
},
{
"color": "red",
"value": 80
}
]
}
},
"overrides": []
},
"gridPos": {
"h": 8,
"w": 5,
"x": 19,
"y": 14
},
"id": 129,
"options": {
"legend": {
"calcs": [
"last",
"min",
"max"
],
"displayMode": "table",
"placement": "bottom"
},
"tooltip": {
"mode": "multi"
}
},
"targets": [
{
"exemplar": true,
"expr": "increase(tomcat_processingtime_total{agenttype=\"jmx-agent\", instance=\"$instance\", job=\"jmx\", servicetype=\"$service\"}[1m]) / increase(tomcat_requestcount_total{agenttype=\"jmx-agent\", instance=\"$instance\", job=\"jmx\", servicetype=\"$service\"}[1m]) ",
"hide": false,
"interval": "",
"legendFormat": "平均请求时间",
"refId": "B"
},
{
"exemplar": true,
"expr": "tomcat_maxtime_total{agenttype=\"jmx-agent\", instance=\"$instance\", job=\"jmx\", servicetype=\"$service\"}",
"interval": "",
"legendFormat": "最慢请求时间",
"refId": "A"
}
],
"title": "请求时间(ms)",
"type": "timeseries"
},
{
"collapsed": false,
"datasource": null,
"gridPos": {
"h": 1,
"w": 24,
"x": 0,
"y": 22
},
"id": 82,
"panels": [],
"title": "内存使用",
"type": "row"
},
{
"aliasColors": {},
"bars": false,
"dashLength": 10,
"dashes": false,
"datasource": "$datasource",
"fill": 1,
"fillGradient": 0,
"gridPos": {
"h": 9,
"w": 12,
"x": 0,
"y": 23
},
"hiddenSeries": false,
"id": 8,
"legend": {
"alignAsTable": true,
"avg": true,
"current": true,
"max": true,
"min": true,
"rightSide": false,
"show": true,
"sortDesc": true,
"total": false,
"values": true
},
"lines": true,
"linewidth": 1,
"links": [],
"maxPerRow": 2,
"nullPointMode": "null",
"options": {
"alertThreshold": true
},
"percentage": false,
"pluginVersion": "8.1.5",
"pointradius": 5,
"points": false,
"renderer": "flot",
"repeat": "memarea",
"repeatDirection": "h",
"seriesOverrides": [
{
"$$hashKey": "object:211",
"alias": "Usage %",
"bars": true,
"color": "#6d1f62",
"legend": false,
"lines": false,
"yaxis": 2,
"zindex": -1
}
],
"spaceLength": 10,
"stack": false,
"steppedLine": false,
"targets": [
{
"exemplar": true,
"expr": "jvm_memory_bytes_used{area=\"$memarea\",job=\"$job\",instance=\"$instance\",servicetype=\"$service\"}",
"format": "time_series",
"interval": "",
"intervalFactor": 1,
"legendFormat": "Used",
"refId": "A"
},
{
"exemplar": true,
"expr": " jvm_memory_bytes_max{area=\"$memarea\",job=\"$job\",instance=\"$instance\",servicetype=\"$service\"}",
"format": "time_series",
"interval": "",
"intervalFactor": 1,
"legendFormat": "Max",
"refId": "B"
}
],
"thresholds": [],
"timeFrom": null,
"timeRegions": [],
"timeShift": null,
"title": "Memory area [$memarea]",
"tooltip": {
"shared": true,
"sort": 0,
"value_type": "individual"
},
"type": "graph",
"xaxis": {
"buckets": null,
"mode": "time",
"name": null,
"show": true,
"values": []
},
"yaxes": [
{
"format": "bytes",
"label": null,
"logBase": 1,
"max": null,
"min": null,
"show": true
},
{
"format": "percentunit",
"label": "",
"logBase": 1,
"max": "1",
"min": "0",
"show": true
}
],
"yaxis": {
"align": false,
"alignLevel": null
}
},
{
"aliasColors": {},
"bars": false,
"dashLength": 10,
"dashes": false,
"datasource": "$datasource",
"fill": 1,
"fillGradient": 0,
"gridPos": {
"h": 9,
"w": 12,
"x": 0,
"y": 32
},
"hiddenSeries": false,
"id": 2,
"legend": {
"alignAsTable": true,
"avg": true,
"current": true,
"max": true,
"min": true,
"rightSide": false,
"show": true,
"sort": "current",
"sortDesc": true,
"total": false,
"values": true
},
"lines": true,
"linewidth": 1,
"links": [],
"maxPerRow": 2,
"nullPointMode": "null",
"options": {
"alertThreshold": true
},
"percentage": false,
"pluginVersion": "8.1.5",
"pointradius": 5,
"points": false,
"renderer": "flot",
"repeat": "mempool",
"repeatDirection": "h",
"seriesOverrides": [],
"spaceLength": 10,
"stack": false,
"steppedLine": false,
"targets": [
{
"exemplar": true,
"expr": "jvm_memory_pool_bytes_max{pool=\"$mempool\",job=\"$job\",instance=\"$instance\",servicetype=\"$service\"}",
"format": "time_series",
"interval": "",
"intervalFactor": 1,
"legendFormat": "Max",
"metric": "jvm_memory_bytes_used",
"refId": "B",
"step": 5
},
{
"exemplar": true,
"expr": "jvm_memory_pool_bytes_used{pool=\"$mempool\",job=\"$job\",instance=\"$instance\",servicetype=\"$service\"}",
"format": "time_series",
"interval": "",
"intervalFactor": 1,
"legendFormat": "Used",
"metric": "jvm_memory_bytes_used",
"refId": "A",
"step": 5
},
{
"exemplar": true,
"expr": "jvm_memory_pool_bytes_committed{pool=\"$mempool\",job=\"$job\",instance=\"$instance\",servicetype=\"$service\"}",
"format": "time_series",
"interval": "",
"intervalFactor": 1,
"legendFormat": "Committed",
"metric": "jvm_memory_bytes_used",
"refId": "C",
"step": 5
}
],
"thresholds": [],
"timeFrom": null,
"timeRegions": [],
"timeShift": null,
"title": "Memory pool [$mempool]",
"tooltip": {
"shared": true,
"sort": 0,
"value_type": "individual"
},
"type": "graph",
"xaxis": {
"buckets": null,
"mode": "time",
"name": null,
"show": true,
"values": []
},
"yaxes": [
{
"format": "bytes",
"label": null,
"logBase": 1,
"max": null,
"min": null,
"show": true
},
{
"format": "short",
"label": null,
"logBase": 1,
"max": null,
"min": null,
"show": true
}
],
"yaxis": {
"align": false,
"alignLevel": null
}
},
{
"collapsed": false,
"datasource": null,
"gridPos": {
"h": 1,
"w": 24,
"x": 0,
"y": 59
},
"id": 74,
"panels": [],
"title": "gc信息",
"type": "row"
},
{
"aliasColors": {},
"bars": false,
"dashLength": 10,
"dashes": false,
"datasource": "$datasource",
"decimals": 0,
"fill": 1,
"fillGradient": 0,
"gridPos": {
"h": 9,
"w": 12,
"x": 0,
"y": 60
},
"hiddenSeries": false,
"id": 6,
"legend": {
"alignAsTable": true,
"avg": false,
"current": true,
"max": true,
"min": true,
"show": true,
"total": false,
"values": true
},
"lines": true,
"linewidth": 1,
"links": [],
"nullPointMode": "null",
"options": {
"alertThreshold": true
},
"percentage": false,
"pluginVersion": "8.1.5",
"pointradius": 5,
"points": false,
"renderer": "flot",
"seriesOverrides": [],
"spaceLength": 10,
"stack": false,
"steppedLine": false,
"targets": [
{
"exemplar": true,
"expr": "increase(jvm_gc_collection_seconds_count{job=\"$job\",instance=~\"$instance\",servicetype=\"$service\"}[$__interval])",
"format": "time_series",
"interval": "60s",
"intervalFactor": 1,
"legendFormat": "{{gc}}",
"metric": "",
"refId": "A",
"step": 10
}
],
"thresholds": [],
"timeFrom": null,
"timeRegions": [],
"timeShift": null,
"title": "GC count increase",
"tooltip": {
"shared": true,
"sort": 0,
"value_type": "individual"
},
"type": "graph",
"xaxis": {
"buckets": null,
"mode": "time",
"name": null,
"show": true,
"values": []
},
"yaxes": [
{
"decimals": 0,
"format": "short",
"label": null,
"logBase": 1,
"max": null,
"min": null,
"show": true
},
{
"format": "short",
"label": null,
"logBase": 1,
"max": null,
"min": null,
"show": false
}
],
"yaxis": {
"align": false,
"alignLevel": null
}
},
{
"aliasColors": {},
"bars": false,
"dashLength": 10,
"dashes": false,
"datasource": "$datasource",
"decimals": 0,
"fill": 1,
"fillGradient": 0,
"gridPos": {
"h": 9,
"w": 12,
"x": 12,
"y": 60
},
"hiddenSeries": false,
"id": 5,
"legend": {
"alignAsTable": true,
"avg": false,
"current": true,
"max": true,
"min": true,
"rightSide": false,
"show": true,
"total": false,
"values": true
},
"lines": true,
"linewidth": 1,
"links": [],
"nullPointMode": "null",
"options": {
"alertThreshold": true
},
"percentage": false,
"pluginVersion": "8.1.5",
"pointradius": 5,
"points": false,
"renderer": "flot",
"seriesOverrides": [],
"spaceLength": 10,
"stack": false,
"steppedLine": false,
"targets": [
{
"exemplar": true,
"expr": "increase(jvm_gc_collection_seconds_sum{job=\"$job\",instance=~\"$instance\",servicetype=\"$service\"}[$__interval])",
"format": "time_series",
"interval": "60s",
"intervalFactor": 1,
"legendFormat": "{{gc}}",
"metric": "jvm_gc_collection_seconds_sum",
"refId": "A",
"step": 10
}
],
"thresholds": [],
"timeFrom": null,
"timeRegions": [],
"timeShift": null,
"title": "GC time",
"tooltip": {
"shared": true,
"sort": 0,
"value_type": "individual"
},
"type": "graph",
"xaxis": {
"buckets": null,
"mode": "time",
"name": null,
"show": true,
"values": []
},
"yaxes": [
{
"format": "s",
"label": null,
"logBase": 1,
"max": null,
"min": null,
"show": true
},
{
"format": "short",
"label": null,
"logBase": 1,
"max": null,
"min": null,
"show": false
}
],
"yaxis": {
"align": false,
"alignLevel": null
}
},
{
"collapsed": false,
"datasource": null,
"gridPos": {
"h": 1,
"w": 24,
"x": 0,
"y": 69
},
"id": 66,
"panels": [],
"title": "线程信息",
"type": "row"
},
{
"datasource": "$datasource",
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"axisLabel": "",
"axisPlacement": "auto",
"barAlignment": 0,
"drawStyle": "line",
"fillOpacity": 10,
"gradientMode": "none",
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
},
"lineInterpolation": "linear",
"lineWidth": 1,
"pointSize": 5,
"scaleDistribution": {
"type": "linear"
},
"showPoints": "never",
"spanNulls": true,
"stacking": {
"group": "A",
"mode": "none"
},
"thresholdsStyle": {
"mode": "off"
}
},
"decimals": 0,
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
},
{
"color": "red",
"value": 80
}
]
},
"unit": "short"
},
"overrides": []
},
"gridPos": {
"h": 6,
"w": 12,
"x": 0,
"y": 70
},
"id": 3,
"links": [],
"options": {
"legend": {
"calcs": [
"lastNotNull",
"max"
],
"displayMode": "table",
"placement": "bottom"
},
"tooltip": {
"mode": "multi"
}
},
"pluginVersion": "8.1.5",
"targets": [
{
"exemplar": true,
"expr": "jvm_threads_current{job=\"$job\",instance=\"$instance\",servicetype=\"$service\"}",
"format": "time_series",
"interval": "",
"intervalFactor": 5,
"legendFormat": "当前线程",
"metric": "jvm_threads_current",
"refId": "A",
"step": 10
},
{
"exemplar": true,
"expr": "jvm_threads_daemon{job=\"$job\",instance=\"$instance\",servicetype=\"$service\"}",
"format": "time_series",
"interval": "",
"intervalFactor": 5,
"legendFormat": "守护线程",
"metric": "jvm_threads_daemon",
"refId": "B",
"step": 10
},
{
"exemplar": true,
"expr": "jvm_threads_peak{job=\"$job\",instance=\"$instance\",servicetype=\"$service\"}",
"format": "time_series",
"interval": "",
"intervalFactor": 1,
"legendFormat": "峰值线程",
"refId": "C"
}
],
"timeFrom": null,
"timeShift": null,
"title": "线程水位",
"type": "timeseries"
},
{
"datasource": null,
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"axisLabel": "",
"axisPlacement": "auto",
"barAlignment": 0,
"drawStyle": "line",
"fillOpacity": 0,
"gradientMode": "none",
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
},
"lineInterpolation": "linear",
"lineStyle": {
"fill": "solid"
},
"lineWidth": 1,
"pointSize": 5,
"scaleDistribution": {
"type": "linear"
},
"showPoints": "never",
"spanNulls": false,
"stacking": {
"group": "A",
"mode": "none"
},
"thresholdsStyle": {
"mode": "off"
}
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
},
{
"color": "red",
"value": 80
}
]
}
},
"overrides": []
},
"gridPos": {
"h": 6,
"w": 12,
"x": 12,
"y": 70
},
"id": 49,
"options": {
"legend": {
"calcs": [
"last",
"max"
],
"displayMode": "table",
"placement": "bottom"
},
"tooltip": {
"mode": "multi"
}
},
"targets": [
{
"exemplar": true,
"expr": "jvm_threads_deadlocked{job=\"$job\",instance=\"$instance\",servicetype=\"$service\"}",
"interval": "",
"legendFormat": "线程死锁",
"refId": "A"
}
],
"title": "线程死锁监测",
"type": "timeseries"
},
{
"datasource": null,
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"axisLabel": "",
"axisPlacement": "auto",
"barAlignment": 0,
"drawStyle": "line",
"fillOpacity": 0,
"gradientMode": "none",
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
},
"lineInterpolation": "linear",
"lineWidth": 1,
"pointSize": 5,
"scaleDistribution": {
"type": "linear"
},
"showPoints": "auto",
"spanNulls": false,
"stacking": {
"group": "A",
"mode": "none"
},
"thresholdsStyle": {
"mode": "off"
}
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
},
{
"color": "red",
"value": 80
}
]
}
},
"overrides": []
},
"gridPos": {
"h": 8,
"w": 12,
"x": 0,
"y": 76
},
"id": 130,
"options": {
"legend": {
"calcs": [
"last",
"min",
"max"
],
"displayMode": "table",
"placement": "bottom"
},
"tooltip": {
"mode": "multi"
}
},
"targets": [
{
"exemplar": true,
"expr": "tomcat_threadpool_connectioncount{agenttype=\"jmx-agent\", instance=\"$instance\", job=\"jmx\", servicetype=\"$service\"}",
"interval": "",
"legendFormat": "connectioncount",
"refId": "A"
},
{
"exemplar": true,
"expr": "tomcat_threadpool_currentthreadcount{agenttype=\"jmx-agent\", instance=\"$instance\", job=\"jmx\", servicetype=\"$service\"}",
"hide": false,
"interval": "",
"legendFormat": "currentthreadcount",
"refId": "B"
},
{
"exemplar": true,
"expr": "tomcat_threadpool_currentthreadsbusy{agenttype=\"jmx-agent\", instance=\"$instance\", job=\"jmx\", servicetype=\"$service\"}",
"hide": false,
"interval": "",
"legendFormat": "currentthreadsbusy",
"refId": "C"
},
{
"exemplar": true,
"expr": "tomcat_threadpool_keepalivecount{agenttype=\"jmx-agent\", instance=\"$instance\", job=\"jmx\", servicetype=\"$service\"}",
"hide": false,
"interval": "",
"legendFormat": "keepalivecount",
"refId": "D"
},
{
"exemplar": true,
"expr": "tomcat_threadpool_pollerthreadcount{agenttype=\"jmx-agent\", instance=\"$instance\", job=\"jmx\", servicetype=\"$service\"}",
"hide": false,
"interval": "",
"legendFormat": "pollerthreadcount",
"refId": "E"
}
],
"title": "tomcat线程状态",
"type": "timeseries"
},
{
"datasource": null,
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"axisLabel": "",
"axisPlacement": "auto",
"barAlignment": 0,
"drawStyle": "line",
"fillOpacity": 0,
"gradientMode": "none",
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
},
"lineInterpolation": "linear",
"lineWidth": 1,
"pointSize": 5,
"scaleDistribution": {
"type": "linear"
},
"showPoints": "never",
"spanNulls": false,
"stacking": {
"group": "A",
"mode": "none"
},
"thresholdsStyle": {
"mode": "off"
}
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
},
{
"color": "red",
"value": 80
}
]
}
},
"overrides": []
},
"gridPos": {
"h": 8,
"w": 12,
"x": 12,
"y": 76
},
"id": 50,
"options": {
"legend": {
"calcs": [
"last",
"max"
],
"displayMode": "table",
"placement": "bottom"
},
"tooltip": {
"mode": "multi"
}
},
"targets": [
{
"exemplar": true,
"expr": "jvm_threads_state{job=\"$job\",instance=\"$instance\",servicetype=\"$service\", state=\"BLOCKED\"}",
"interval": "",
"legendFormat": "阻塞线程",
"refId": "A"
},
{
"exemplar": true,
"expr": "jvm_threads_state{job=\"$job\",instance=\"$instance\",servicetype=\"$service\", state=\"NEW\"}",
"hide": false,
"interval": "",
"legendFormat": "新建线程",
"refId": "B"
},
{
"exemplar": true,
"expr": "jvm_threads_state{job=\"$job\",instance=\"$instance\",servicetype=\"$service\", state=\"RUNNABLE\"}",
"hide": false,
"interval": "",
"legendFormat": "运行线程",
"refId": "C"
},
{
"exemplar": true,
"expr": "jvm_threads_state{job=\"$job\",instance=\"$instance\",servicetype=\"$service\", state=\"TERMINATED\"}",
"hide": false,
"interval": "",
"legendFormat": "终止线程",
"refId": "E"
},
{
"exemplar": true,
"expr": "jvm_threads_state{job=\"$job\",instance=\"$instance\",servicetype=\"$service\", state=\"WAITING\"}",
"hide": false,
"interval": "",
"legendFormat": "等待线程",
"refId": "F"
},
{
"exemplar": true,
"expr": "jvm_threads_state{job=\"$job\",instance=\"$instance\",servicetype=\"$service\", state=\"TIMED_WAITING\"}",
"hide": false,
"interval": "",
"legendFormat": "指定时间的等待线程",
"refId": "D"
}
],
"title": "线程状态",
"type": "timeseries"
},
{
"collapsed": false,
"datasource": null,
"gridPos": {
"h": 1,
"w": 24,
"x": 0,
"y": 84
},
"id": 58,
"panels": [],
"title": "nmt内存泄漏追踪",
"type": "row"
},
{
"datasource": null,
"description": "",
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"axisLabel": "",
"axisPlacement": "auto",
"barAlignment": 0,
"drawStyle": "line",
"fillOpacity": 0,
"gradientMode": "none",
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
},
"lineInterpolation": "linear",
"lineWidth": 1,
"pointSize": 5,
"scaleDistribution": {
"type": "linear"
},
"showPoints": "never",
"spanNulls": true,
"stacking": {
"group": "A",
"mode": "none"
},
"thresholdsStyle": {
"mode": "off"
}
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
},
{
"color": "red",
"value": 80
}
]
}
},
"overrides": []
},
"gridPos": {
"h": 17,
"w": 24,
"x": 0,
"y": 85
},
"id": 99,
"options": {
"legend": {
"calcs": [
"last",
"max"
],
"displayMode": "table",
"placement": "bottom"
},
"tooltip": {
"mode": "multi"
}
},
"targets": [
{
"exemplar": true,
"expr": "jvminfo_Class{host=\"$host\",servicetype=\"$service\"} / 1024",
"hide": false,
"interval": "",
"legendFormat": "jvminfo_Class",
"refId": "C"
},
{
"exemplar": true,
"expr": "jvminfo_Thread{host=\"$host\",servicetype=\"$service\"} / 1024",
"hide": false,
"interval": "",
"legendFormat": "jvminfo_Thread",
"refId": "D"
},
{
"exemplar": true,
"expr": "jvminfo_Code{host=\"$host\",servicetype=\"$service\"} / 1024",
"hide": false,
"interval": "",
"legendFormat": "jvminfo_Code",
"refId": "E"
},
{
"exemplar": true,
"expr": "jvminfo_GC{host=\"$host\",servicetype=\"$service\"} / 1024",
"hide": false,
"interval": "",
"legendFormat": "jvminfo_GC",
"refId": "F"
},
{
"exemplar": true,
"expr": "jvminfo_Compiler{host=\"$host\",servicetype=\"$service\"} / 1024",
"hide": false,
"interval": "",
"legendFormat": "jvminfo_Compiler",
"refId": "G"
},
{
"exemplar": true,
"expr": "jvminfo_Internal{host=\"$host\",servicetype=\"$service\"} / 1024",
"hide": false,
"interval": "",
"legendFormat": "jvminfo_Internal",
"refId": "H"
},
{
"exemplar": true,
"expr": "jvminfo_Symbol{host=\"$host\",servicetype=\"$service\"} / 1024",
"hide": false,
"interval": "",
"legendFormat": "jvminfo_Symbol",
"refId": "I"
},
{
"exemplar": true,
"expr": "jvminfo_Total{host=\"$host\",servicetype=\"$service\"} / 1024",
"hide": false,
"interval": "",
"legendFormat": "jvminfo_Total",
"refId": "A"
},
{
"exemplar": true,
"expr": "jvminfo_Heap{host=\"$host\",servicetype=\"$service\"} / 1024",
"hide": false,
"interval": "",
"legendFormat": "jvminfo_Heap",
"refId": "B"
},
{
"exemplar": true,
"expr": "jvminfo_classes{host=\"$host\",servicetype=\"$service\"} / 1024",
"hide": false,
"interval": "",
"legendFormat": "jvminfo_classes",
"refId": "J"
}
],
"title": "内存分配信息（MB）",
"type": "timeseries"
}
],
"refresh": "1m",
"schemaVersion": 30,
"style": "dark",
"tags": [
"JVM"
],
"templating": {
"list": [
{
"current": {
"selected": false,
"text": "default",
"value": "default"
},
"description": null,
"error": null,
"hide": 2,
"includeAll": false,
"label": null,
"multi": false,
"name": "datasource",
"options": [],
"query": "prometheus",
"queryValue": "",
"refresh": 1,
"regex": "",
"skipUrlSync": false,
"type": "datasource"
},
{
"current": {
"selected": true,
"text": "jmx",
"value": "jmx"
},
"description": null,
"error": null,
"hide": 2,
"label": "job",
"name": "job",
"options": [
{
"selected": true,
"text": "jmx",
"value": "jmx"
}
],
"query": "jmx",
"skipUrlSync": false,
"type": "textbox"
},
{
"allValue": null,
"current": {
"selected": true,
"text": [
"All"
],
"value": [
"$__all"
]
},
"datasource": "$datasource",
"definition": "",
"description": null,
"error": null,
"hide": 2,
"includeAll": true,
"label": null,
"multi": true,
"name": "mempool",
"options": [],
"query": {
"query": "label_values(jvm_memory_pool_bytes_max{job=\"$job\", instance=\"$instance\"}, pool)",
"refId": "Prometheus-mempool-Variable-Query"
},
"refresh": 1,
"regex": "",
"skipUrlSync": false,
"sort": 1,
"tagValuesQuery": "",
"tagsQuery": "",
"type": "query",
"useTags": false
},
{
"allValue": null,
"current": {
"selected": true,
"text": [
"All"
],
"value": [
"$__all"
]
},
"datasource": "$datasource",
"definition": "label_values(jvm_memory_bytes_used{job=\"$job\", instance=\"$instance\"}, area)",
"description": null,
"error": null,
"hide": 2,
"includeAll": true,
"label": null,
"multi": true,
"name": "memarea",
"options": [],
"query": {
"query": "label_values(jvm_memory_bytes_used{job=\"$job\", instance=\"$instance\"}, area)",
"refId": "StandardVariableQuery"
},
"refresh": 1,
"regex": "",
"skipUrlSync": false,
"sort": 0,
"tagValuesQuery": "",
"tagsQuery": "",
"type": "query",
"useTags": false
},
{
"allValue": ".*",
"current": {
"selected": false,
"text": "etown-member-admin",
"value": "etown-member-admin"
},
"datasource": null,
"definition": "label_values(jvm_info{job=\"$job\"},servicetype)",
"description": null,
"error": null,
"hide": 0,
"includeAll": false,
"label": "service",
"multi": false,
"name": "service",
"options": [],
"query": {
"query": "label_values(jvm_info{job=\"$job\"},servicetype)",
"refId": "StandardVariableQuery"
},
"refresh": 1,
"regex": "",
"skipUrlSync": false,
"sort": 1,
"tagValuesQuery": "",
"tagsQuery": "",
"type": "query",
"useTags": false
},
{
"allValue": ".*",
"current": {
"selected": false,
"text": "10.148.108.43:19013",
"value": "10.148.108.43:19013"
},
"datasource": "$datasource",
"definition": "label_values(jvm_info{job=\"$job\",servicetype=\"$service\"},instance)",
"description": null,
"error": null,
"hide": 0,
"includeAll": false,
"label": "instance",
"multi": false,
"name": "instance",
"options": [],
"query": {
"query": "label_values(jvm_info{job=\"$job\",servicetype=\"$service\"},instance)",
"refId": "StandardVariableQuery"
},
"refresh": 1,
"regex": "",
"skipUrlSync": false,
"sort": 1,
"tagValuesQuery": "",
"tagsQuery": "",
"type": "query",
"useTags": false
},
{
"allValue": ".*",
"current": {
"selected": false,
"text": "L-Jdy-prod-yzsyc-app-a12",
"value": "L-Jdy-prod-yzsyc-app-a12"
},
"datasource": "$datasource",
"definition": "label_values(jvminfo_Class{servicetype=\"$service\"},host)",
"description": null,
"error": null,
"hide": 0,
"includeAll": false,
"label": "host",
"multi": false,
"name": "host",
"options": [],
"query": {
"query": "label_values(jvminfo_Class{servicetype=\"$service\"},host)",
"refId": "StandardVariableQuery"
},
"refresh": 1,
"regex": "",
"skipUrlSync": false,
"sort": 1,
"tagValuesQuery": "",
"tagsQuery": "",
"type": "query",
"useTags": false
}
]
},
"time": {
"from": "now-5m",
"to": "now"
},
"timepicker": {
"refresh_intervals": [
"5s",
"10s",
"30s",
"1m",
"5m"
],
"time_options": [
"5m",
"15m",
"1h",
"6h",
"12h",
"24h",
"2d",
"7d",
"30d"
]
},
"timezone": "",
"title": "应用监控",
"uid": "chanjarster-jvm-dashboard",
"version": 36
}
