helm部署kube-promethues
访问https://artifacthub.io/，搜索prometheus
找到kube-prometheus
拉取仓库
[root@k8s-Master-01 ~]#helm repo add bitnami https://charts.bitnami.com/bitnami
[root@k8s-Master-01 ~]#helm repo list
NAME URL
my-repo https://charts.bitnami.com/bitnami
拉取文档，借助文档在k8s上部署prometheus-server
[root@k8s-Master-01 ~]#git clone https://github.com/iKubernetes/learning-k8s
进入文档目录，查看写好的yaml文件
[root@k8s-Master-01 ~]#cd learning-k8s/helm/kube-prometheus/
[root@k8s-Master-01 kube-prometheus]#cat kube-prometheus-values.yaml
global:
imageRegistry: ""
storageClass: "nfs-csi"
clusterDomain: cluster.local
operator:
enabled: true
image:
registry: docker.io
repository: bitnami/prometheus-operator
tag: 0.60.1-debian-11-r9
prometheus:
enabled: true
replicaCount: 2
image:
registry: docker.io
repository: bitnami/prometheus
tag: 2.39.1-debian-11-r10
ingress:
enabled: true
pathType: Prefix
apiVersion: ""
hostname: prometheus.magedu.com
path: /
annotations: {}
ingressClassName: "nginx"
tls: false
selfSigned: false
externalUrl: ""
persistence:
enabled: true
storageClass: ""
accessModes:
- ReadWriteOnce
size: 8Gi
annotations: {}
priorityClassName: ""
thanos:
create: false
image:
registry: docker.io
repository: bitnami/thanos
tag: 0.28.1-scratch-r0
ingress:
enabled: false
pathType: Prefix
hostname: thanos.magedu.com
path: /
annotations: {}
ingressClassName: "nginx"
tls: false
selfSigned: false
alertmanager:
enabled: true
replicaCount: 2
image:
registry: docker.io
repository: bitnami/alertmanager
tag: 0.24.0-debian-11-r55
ingress:
enabled: true
pathType: Prefix
hostname: alertmanager.magedu.com
path: /
annotations: {}
ingressClassName: "nginx"
tls: false
selfSigned: false
persistence:
enabled:true
storageClass: ""
accessModes:
- ReadWriteOnce
size: 8Gi
annotations: {}
exporters:
node-exporter:
enabled: true
kube-state-metrics:
enabled: true
node-exporter:
service:
labels:
jobLabel: node-exporter
serviceMonitor:
enabled: true
jobLabel: jobLabel
extraArgs:
collector.filesystem.ignored-mount-points: "^/(dev|proc|sys|var/lib/docker/.+)($|/)"
collector.filesystem.ignored-fs-types: "^(autofs|binfmt_misc|cgroup|configfs|debugfs|devpts|devtmpfs|fusectl|hugetlbfs|mqueue|overlay|proc|procfs|pstore|rpc_pipefs|securityfs|sysfs|tracefs)$"
kube-state-metrics:
serviceMonitor:
enabled: true
kubelet:
enabled: true
namespace: kube-system
blackboxExporter:
enabled: true
image:
registry: docker.io
repository: bitnami/blackbox-exporter
tag: 0.22.0-debian-11-r32
configuration: |
"modules":
"http_2xx":
"http":
"preferred_ip_protocol": "ip4"
"prober": "http"
"http_post_2xx":
"http":
"method": "POST"
"preferred_ip_protocol": "ip4"
"prober": "http"
"irc_banner":
"prober": "tcp"
"tcp":
"preferred_ip_protocol": "ip4"
"query_response":
- "send": "NICK prober"
- "send": "USER prober prober prober :prober"
- "expect": "PING :([^ ]+)"
"send": "PONG ${1}"
- "expect": "^:[^ ]+ 001"
"pop3s_banner":
"prober": "tcp"
"tcp":
"preferred_ip_protocol": "ip4"
"query_response":
- "expect": "^+OK"
"tls": true
"tls_config":
"insecure_skip_verify": false
"ssh_banner":
"prober": "tcp"
"tcp":
"preferred_ip_protocol": "ip4"
"query_response":
- "expect": "^SSH-2.0-"
"tcp_connect":
"prober": "tcp"
"tcp":
"preferred_ip_protocol": "ip4"
kubeApiServer:
enabled: true
kubeControllerManager:
enabled: false
kubeScheduler:
enabled: false
coreDns:
enabled: true
kubeProxy:
enabled: false
rbac:
create: true
pspEnabled: true
yaml文件释义
1.storageClass: "nfs-csi" #网络存储——nfs-csi
2.prometheus借助于operator部署prometheus实例，并且指明prometheus运行两个副本，并且通过ingress开放出去，主机名为prometheus.magedu.com，并作持久化保存
3.启用http协议对外提供服务
4.alertmanager借助于operator部署alertmanager实例，并且指明alertmanager运行两个副本，并通过ingress开放出去，主机名为alertmanager.magedu.com，并作持久化保存
5.每个节点部署node-exporter和kube-state-matrics
6.在node-exporter监控各个服务等
7.kube-state-matrics和kubelet也纳入到监控体系
8.启用blackbox监控，并提供默认配置问文件
9.监控kubeapiserver、coreDNS、kubeproxy、rbak
运行prometheus-server
[root@k8s-Master-01 helm]#kubectl create namespace prom
[root@k8s-Master-01 helm]#kubectl create namespace ingress-nginx
运行prometheus-server
[root@k8s-Master-01 kube-prometheus]#pwd
/root/learning-k8s/helm/kube-prometheus
[root@k8s-Master-01 kube-prometheus]#helm install prometheus -f kube-prometheus-values.yaml bitnami/kube-prometheus -n prom
[root@k8s-Master-01 ~]#kubectl get ingress -n prom
NAME CLASS HOSTS ADDRESS PORTS AGE
prometheus-kube-prometheus-alertmanager nginx alertmanager.magedu.com 10.0.0.200 80 10m
prometheus-kube-prometheus-prometheus nginx prometheus.magedu.com 10.0.0.200 80 10m
[root@k8s-Master-01 kube-prometheus]#kubectl get cm -n prom
NAME DATA AGE
kube-root-ca.crt 1 10m
prometheus-kube-prometheus-blackbox-exporter 1 7m46s
prometheus-kube-prometheus-operator 1 7m46s
prometheus-prometheus-kube-prometheus-prometheus-rulefiles-0 0 76s
[root@k8s-Master-01 kube-prometheus]#kubectl get pods -n prom
NAME READY STATUS RESTARTS AGE
alertmanager-prometheus-kube-prometheus-alertmanager-0 2/2 Running 1 (19m ago) 22m
alertmanager-prometheus-kube-prometheus-alertmanager-1 2/2 Running 0 22m
prometheus-kube-prometheus-blackbox-exporter-677f8dbc57-rpltp 1/1 Running 0 30m
prometheus-kube-prometheus-operator-b66d587cf-lrt79 1/1 Running 0 30m
prometheus-kube-state-metrics-6cfb9bb977-2t7sm 1/1 Running 0 30m
prometheus-node-exporter-j275g 1/1 Running 0 30m
prometheus-node-exporter-q978g 1/1 Running 0 30m
prometheus-node-exporter-xdrql 1/1 Running 0 30m
prometheus-prometheus-kube-prometheus-prometheus-0 2/2 Running 0 22m
prometheus-prometheus-kube-prometheus-prometheus-1 2/2 Running 0 22m
在物理机上对域名及IP进行解析后进行访问
其告警规则和告警方式可以基于configmap创建配置上去
告警规则文档：
https://github.com/iKubernetes/learning-prometheus/blob/master/08-prometheus-components-compose/prometheus-server/prometheus/rules/kubernetes-rules.yml
部署prometheus-adapter
目的：把prometheus上的指标转换成kubernetes上的格式并予以暴露。
访问https://artifacthub.io/，搜索prometheus-adapter
拉取仓库
[root@k8s-Master-01 kube-prometheus]#helm repo add prometheus-community https://prometheus-community.github.io/helm-charts
"prometheus-community" has been added to your repositories
[root@k8s-Master-01 kube-prometheus]#helm repo list
NAME URL
my-repo https://charts.bitnami.com/bitnami
prometheus-community https://prometheus-community.github.io/helm-charts
把仓库中关于prometheus-adapter的默认规则定义文件并修改
[root@k8s-Master-01 kube-prometheus]#helm show values prometheus-community/prometheus-adapter > prometheus-adapter.yml
修改其文件
[root@k8s-Master-01 kube-prometheus]#cat prometheus-adapter.yml
affinity: {}
topologySpreadConstraints: []
image:
repository: k8s.gcr.io/prometheus-adapter/prometheus-adapter
tag: v0.10.0
pullPolicy: IfNotPresent
logLevel: 4
metricsRelistInterval: 1m
listenPort: 6443
# User to run adapter container as
runAsUser: 10001
rometheus:
# Value is templated
url: http://prometheus-kube-prometheus-prometheus.prom.svc.cluster.local
port: 9090
path: ""
replicas: 1
# k8s 1.21 needs fsGroup to be set for non root deployments
# ref: https://github.com/kubernetes/kubernetes/issues/70679
podSecurityContext:
fsGroup: 10001
rbac:
# Specifies whether RBAC resources should be created
create: true
rules:
default: true
custom:
- seriesQuery: 'http_requests_total{kubernetes_namespace!="",kubernetes_pod_name!=""}'
resources:
overrides:
kubernetes_namespace: {resource: "namespace"}
kubernetes_pod_name: {resource: "pod"}
name:
matches: "^(.*)_total"
as: "${1}_per_second"
metricsQuery: 'rate(<<.Series>>{<<.LabelMatchers>>}[2m])'
# - seriesQuery: '{__name__=~"^some_metric_count$"}'
# resources:
# template: <<.Resource>>
# name:
# matches: ""
# as: "my_custom_metric"
# metricsQuery: sum(<<.Series>>{<<.LabelMatchers>>}) by (<<.GroupBy>>)
# Mounts a configMap with pre-generated rules for use. Overrides the
# default, custom, external and resource entries
existing:
external: []
# - seriesQuery: '{__name__=~"^some_metric_count$"}'
# resources:
# template: <<.Resource>>
# name:
# matches: ""
# as: "my_external_metric"
# metricsQuery: sum(<<.Series>>{<<.LabelMatchers>>}) by (<<.GroupBy>>)
# resource:
# cpu:
# containerQuery: sum(rate(container_cpu_usage_seconds_total{<<.LabelMatchers>>, container!=""}[3m])) by (<<.GroupBy>>)
# nodeQuery: sum(rate(container_cpu_usage_seconds_total{<<.LabelMatchers>>, id='/'}[3m])) by (<<.GroupBy>>)
# resources:
# overrides:
# node:
# resource: node
# namespace:
# resource: namespace
# pod:
# resource: pod
# containerLabel: container
# memory:
# containerQuery: sum(container_memory_working_set_bytes{<<.LabelMatchers>>, container!=""}) by (<<.GroupBy>>)
# nodeQuery: sum(container_memory_working_set_bytes{<<.LabelMatchers>>,id='/'}) by (<<.GroupBy>>)
# resources:
# overrides:
# node:
# resource: node
# namespace:
# resource: namespace
# pod:
# resource: pod
# containerLabel: container
# window: 3m
service:
annotations: {}
port: 443
type: ClusterIP
# clusterIP: 1.2.3.4
tls:
enable: false
ca: |-
# Public CA file that signed the APIService
key: |-
# Private key of the APIService
certificate: |-
# Public key of the APIService
# Any extra arguments
extraArguments: []
# - --tls-private-key-file=/etc/tls/tls.key
# - --tls-cert-file=/etc/tls/tls.crt
# Deployment strategy type
strategy:
type: RollingUpdate
rollingUpdate:
maxUnavailable: 25%
maxSurge: 25%
podDisruptionBudget:
# Specifies if PodDisruptionBudget should be enabled
# When enabled, minAvailable or maxUnavailable should also be defined.
enabled: false
minAvailable:
maxUnavailable: 1
certManager:
enabled: false
caCertDuration: 43800h
certDuration: 8760h
启动
[root@k8s-Master-01 kube-prometheus]#ls
kube-prometheus-values.yaml prometheus-adapter.yml
[root@k8s-Master-01 kube-prometheus]#helm install prometheus-adapter -f prometheus-adapter.yml prometheus-community/prometheus-adapter -n prom
NAME: prometheus-adapter
LAST DEPLOYED: Wed Dec 7 00:10:52 2022
NAMESPACE: prom
STATUS: deployed
REVISION: 1
TEST SUITE: None
NOTES:
prometheus-adapter has been deployed.
In a few minutes you should be able to list metrics using the following command(s):
kubectl get --raw /apis/custom.metrics.k8s.io/v1beta1
会出现一个群组
[root@k8s-Master-01 kube-prometheus]#kubectl api-versions
custom.metrics.k8s.io/v1beta1
[root@k8s-Master-01 kube-prometheus]#cat metrics-app.yml
apiVersion: apps/v1
kind: Deployment
metadata:
name: metrics-app
spec:
replicas: 2
selector:
matchLabels:
app: metrics-app
controller: metrics-app
template:
metadata:
labels:
app: metrics-app
controller: metrics-app
annotations:
prometheus.io/scrape: "true"
prometheus.io/port: "80"
prometheus.io/path: "/metrics"
spec:
containers:
- image: ikubernetes/metrics-app
name: metrics-app
ports:
- name: web
containerPort: 80
resources:
requests:
memory: "256Mi"
cpu: "500m"
limits:
memory: "256Mi"
cpu: "500m"
---
apiVersion: v1
kind: Service
metadata:
name: metrics-app
spec:
type: NodePort
ports:
- name: web
port: 80
targetPort: 80
selector:
app: metrics-app
controller: metrics-app
[root@k8s-Master-01 kube-prometheus]#kubectl apply -f metrics-app.yml
deployment.apps/metrics-app created
service/metrics-app created
[root@k8s-Master-01 kube-prometheus]#kubectl get pods
NAME READY STATUS RESTARTS AGE
metrics-app-97fbfb5b4-8mdjb 1/1 Running 0 97m
metrics-app-97fbfb5b4-g8rs5 1/1 Running 0 97m
[root@k8s-Master-01 kube-prometheus]#curl 10.244.4.5
Hello! My name is metrics-app-97fbfb5b4-8mdjb. The last 10 seconds, the average QPS has been 0.1. Total requests served: 1
[root@k8s-Master-01 kube-prometheus]#curl 10.244.4.5
Hello! My name is metrics-app-97fbfb5b4-8mdjb. The last 10 seconds, the average QPS has been 0.2. Total requests served: 2
[root@k8s-Master-01 kube-prometheus]#curl 10.244.4.5
Hello! My name is metrics-app-97fbfb5b4-8mdjb. The last 10 seconds, the average QPS has been 0.3. Total requests served: 3
[root@k8s-Master-01 kube-prometheus]#curl 10.244.4.5/metrics
# HELP http_requests_total The amount of requests in total
# TYPE http_requests_total counter
http_requests_total 7
# HELP http_requests_per_second The amount of requests per second the latest ten seconds
# TYPE http_requests_per_second gauge
http_requests_per_second 0.1
