负载均衡
nginx负载均衡
两台nginx负载均衡
vim /etc/nginx/nginx.conf
stream {
upstream kube-apiserver {
server 192.168.0.11:6443 max_fails=3 fail_timeout=30s;
#server 192.168.0.12:6443 max_fails=3 fail_timeout=30s;
#server 192.168.0.13:6443 max_fails=3 fail_timeout=30s;
}
server {
listen 6443;
proxy_connect_timeout 2s;
proxy_timeout 900s;
proxy_pass kube-apiserver;
}
}
keepalived
nginx检测脚本
vim /data/shell/check_nginx_status.sh
#!/bin/bash
nginx_status=$(ps -ef | grep nginx | grep -v grep | grep -v check | wc -l)
if [ $nginx_status -eq 0 ];then
systemctl stop keepalived.service
fi
master节点keepalived配置（不抢占机制）
vim /etc/keepalived/keepalived.conf
global_defs {
router_id real-server1
}
vrrp_script chk_nginx {
script "/data/shell/check_nginx_status.sh"
interval 2
}
vrrp_instance VI_1 {
state BACKUP
interface ens32
virtual_router_id 151
priority 100
nopreempt
advert_int 5
authentication {
auth_type PASS
auth_pass 1111
}
virtual_ipaddress {
192.168.0.10/24
}
track_script {
chk_nginx
}
}
backup节点keepalived配置
vim /etc/keepalived/keepalived.conf
global_defs {
router_id real-server2
}
vrrp_script chk_nginx {
script "/data/shell/check_nginx_status.sh"
interval 2
}
vrrp_instance VI_1 {
state BACKUP
interface ens32
virtual_router_id 151
priority 50
nopreempt
advert_int 5
authentication {
auth_type PASS
auth_pass 1111
}
virtual_ipaddress {
192.168.0.10/24
}
track_script {
chk_nginx
}
}
k8s节点系统设置
vim /etc/modules-load.d/containerd.conf
overlay
br_netfilter
modprobe overlay
modprobe br_netfilter
vim /etc/sysctl.d/k8s.conf
net.bridge.bridge-nf-call-iptables=1
net.bridge.bridge-nf-call-ip6tables=1
net.ipv4.ip_forward=1
vm.swappiness=0
vm.overcommit_memory=1
vm.panic_on_oom=0
fs.inotify.max_user_instances=8192
fs.inotify.max_user_watches=1048576
fs.file-max=52706963
fs.nr_open=52706963
net.ipv6.conf.all.disable_ipv6=1
net.netfilter.nf_conntrack_max=2310720
sysctl --system
ipvsadm
yum install ipset ipvsadm
modprobe br_netfilter
modprobe overlay
modprobe ip_conntrack
modprobe ip_vs
modprobe ip_vs_rr
modprobe ip_vs_wrr
modprobe ip_vs_sh
modprobe nf_conntrack
lsmod | grep conntrack
lsmod | grep br_netfilt
lsmod | grep overlay
lsmod |egrep "ip_vs|nf_conntrack"
cat > /etc/modules-load.d/kubernetes.conf << EOF
# /etc/modules-load.d/kubernetes.conf
br_netfilter
ip_vs
ip_vs_rr
ip_vs_wrr
ip_vs_sh
nf_conntrack_ipv4
ip_tables
EOF
chmod a+x /etc/modules-load.d/kubernetes.conf
containerd
wget https://github.com/containerd/containerd/releases/download/v1.7.23/cri-containerd-cni-1.7.23-linux-amd64.tar.gz
tar xvf cri-containerd-cni-1.7.23-linux-amd64.tar.gz -C /
wget https://github.com/opencontainers/runc/releases/download/v1.1.5/runc.amd64
cp runc.amd64 /usr/local/sbin/runc
mkdir -p /etc/containerd
containerd config default > /etc/containerd/config.toml
vim /etc/containerd/config.toml
...
SystemdCgroup = true
...
sandbox_image = "registry.cn-beijing.aliyuncs.com/wuxingge123/pause:3.9"
...
[plugins."io.containerd.grpc.v1.cri".registry.mirrors]
[plugins."io.containerd.grpc.v1.cri".registry.mirrors."docker.io"]
endpoint = ["https://p4oudlho.mirror.aliyuncs.com"]
配置私有仓库
[plugins."io.containerd.grpc.v1.cri".registry]
[plugins."io.containerd.grpc.v1.cri".registry.mirrors]
[plugins."io.containerd.grpc.v1.cri".registry.mirrors."docker.io"]
endpoint = ["https://bqr1dr1n.mirror.aliyuncs.com"]
[plugins."io.containerd.grpc.v1.cri".registry.configs]
[plugins."io.containerd.grpc.v1.cri".registry.configs."harbor.k8s.local".tls]
insecure_skip_verify = true
[plugins."io.containerd.grpc.v1.cri".registry.configs."harbor.k8s.local".auth]
username = "admin"
password = "Harbor12345"
启动containerd
systemctl start containerd.service
systemctl enable containerd.service
k8s
安装
https://mirrors.aliyun.com/kubernetes-new/core/stable/v1.30/rpm/x86_64/
vim /etc/yum.repos.d/k8s.repo
[kubernetes]
name=kubernetes
baseurl=https://pkgs.k8s.io/core:/stable:/v1.30/rpm/
enabled=1
gpgcheck=0
在线安装
yum install kubelet-1.30.0 kubectl-1.30.0 kubeadm-1.30.0
离线安装
yum localinstall kubernetes-cni-1.4.0-150500.1.1.x86_64.rpm
yum localinstall cri-tools-1.30.0-150500.1.1.x86_64.rpm
yum localinstall kubeadm-1.30.0-150500.1.1.x86_64.rpm kubectl-1.30.0-150500.1.1.x86_64.rpm kubelet-1.30.0-150500.1.1.x86_64.rpm
配置kubelet
vim /etc/sysconfig/kubelet
KUBELET_EXTRA_ARGS="--cgroup-driver=systemd"
systemctl enable kubelet
下载镜像
kubeadm config images list --kubernetes-version=v1.30.0 --image-repository registry.cn-beijing.aliyuncs.com/wuxingge123
kubeadm config images pull --kubernetes-version=v1.30.0 --image-repository registry.cn-beijing.aliyuncs.com/wuxingge123
calico准备
wget https://docs.projectcalico.org/v3.25/manifests/calico.yaml --no-check-certificate
vim calico.yaml
- name: CALICO_IPV4POOL_CIDR
value: "10.224.0.0/16"
下载calico镜像
ctr -n k8s.io image pull --all-platforms registry.cn-beijing.aliyuncs.com/wuxingge123/cni:v3.25.0
ctr -n k8s.io image pull --all-platforms registry.cn-beijing.aliyuncs.com/wuxingge123/node:v3.25.0
ctr -n k8s.io image pull --all-platforms registry.cn-beijing.aliyuncs.com/wuxingge123/kube-controllers:v3.25.0
k8s init
生成初始化配置文件
kubeadm config print init-defaults > kubeadm-config.yaml
vim kubeadm-config.yaml
apiVersion: kubeadm.k8s.io/v1beta3
bootstrapTokens:
- groups:
- system:bootstrappers:kubeadm:default-node-token
token: abcdef.0123456789abcdef
ttl: 24h0m0s
usages:
- signing
- authentication
kind: InitConfiguration
localAPIEndpoint:
advertiseAddress: 192.168.0.11
bindPort: 6443
nodeRegistration:
criSocket: unix:///var/run/containerd/containerd.sock
imagePullPolicy: IfNotPresent
taints: null
---
apiServer:
timeoutForControlPlane: 4m0s
apiVersion: kubeadm.k8s.io/v1beta3
certificatesDir: /etc/kubernetes/pki
clusterName: kubernetes
controlPlaneEndpoint: 192.168.0.10:6443
controllerManager: {}
dns: {}
etcd:
local:
dataDir: /var/lib/etcd
imageRepository: swr.cn-jl-1.manageone.cloud.cnpc/gsms-project
kind: ClusterConfiguration
kubernetesVersion: 1.30.0
networking:
dnsDomain: cluster.local
podSubnet: 10.224.0.0/16
serviceSubnet: 10.96.0.0/12
scheduler: {}
初始化集群
kubeadm init --config kubeadm-config.yaml --upload-certs
初始化成功显示以下内容
Your Kubernetes control-plane has initialized successfully!
To start using your cluster, you need to run the following as a regular user:
mkdir -p $HOME/.kube
sudo cp -i /etc/kubernetes/admin.conf $HOME/.kube/config
sudo chown $(id -u):$(id -g) $HOME/.kube/config
Alternatively, if you are the root user, you can run:
export KUBECONFIG=/etc/kubernetes/admin.conf
You should now deploy a pod network to the cluster.
Run "kubectl apply -f [podnetwork].yaml" with one of the options listed at:
https://kubernetes.io/docs/concepts/cluster-administration/addons/
You can now join any number of the control-plane node running the following command on each as root:
kubeadm join 192.168.0.10:6443 --token abcdef.0123456789abcdef \
--discovery-token-ca-cert-hash sha256:aac70b668d8010aca7af9a27ad9451468fc985dfd8f52a52025d14b180e16464 \
--control-plane --certificate-key 8ea5c7b87d52438496fce053b1e9788217ffe74ed397d89f6a114e65d06c8826
Please note that the certificate-key gives access to cluster sensitive data, keep it secret!
As a safeguard, uploaded-certs will be deleted in two hours; If necessary, you can use
"kubeadm init phase upload-certs --upload-certs" to reload certs afterward.
Then you can join any number of worker nodes by running the following on each as root:
kubeadm join 192.168.0.10:6443 --token abcdef.0123456789abcdef \
--discovery-token-ca-cert-hash sha256:aac70b668d8010aca7af9a27ad9451468fc985dfd8f52a52025d14b180e16464
配置kubectl客户端
mkdir -p $HOME/.kube
sudo cp -i /etc/kubernetes/admin.conf $HOME/.kube/config
sudo chown $(id -u):$(id -g) $HOME/.kube/config
设置kubectl命令补全
echo "source <(kubectl completion bash)" >> /etc/profile
部署calico
kubectl apply -f calico.yaml
添加master节点
kubeadm join 192.168.0.10:6443 --token abcdef.0123456789abcdef \
--discovery-token-ca-cert-hash sha256:aac70b668d8010aca7af9a27ad9451468fc985dfd8f52a52025d14b180e16464 \
--control-plane --certificate-key 8ea5c7b87d52438496fce053b1e9788217ffe74ed397d89f6a114e65d06c8826
添加node节点
kubeadm join 192.168.0.10:6443 --token abcdef.0123456789abcdef \
--discovery-token-ca-cert-hash sha256:aac70b668d8010aca7af9a27ad9451468fc985dfd8f52a52025d14b180e16464
ingress-nginx v1.11.2
kubectl apply -f https://raw.githubusercontent.com/kubernetes/ingress-nginx/controller-v1.11.2/deploy/static/provider/cloud/deploy.yaml
metrics-server
wget https://github.com/kubernetes-sigs/metrics-server/releases/latest/download/components.yaml
