k8s 1.32 使用docker安装集群-CFANZ编程社区

初始化虚拟机（麒麟v10）

#!/bin/bash

# gzip /etc/yum.repos.d/*.repo && curl -o /etc/yum.repos.d/local.repo http://192.168.3.100/yum/local.repo && yum makecache

[ -f  /etc/init.d/functions ]&& . /etc/init.d/functions

status_check() {
        printf "\033[37m$1\033[0m%-40s\033[32;40m[ $2 ]\033[0m\n"
}

stat() {
        if [ $? -eq 0 ];then
		sleep 0.3
                action "$1" /bin/true
				echo " $1 sucessful " >> /opt/def.txt
        else
		sleep 1
                action "$1" /bin/false
				echo "$1 failed " >> /opt/def.txt
        fi
}

ok() {
        action "$1" /bin/true
}

fail() {
        action "$1" /bin/false
}

HostName() {
        read -p 'pls enter your hostname: ' name
        hostnamectl set-hostname $name
        stat "hostname: $name"

}


yum_update() {
#        \cp -a /etc/yum.repos.d /etc/yum.repos.d_backup &>/dev/null && stat "backup yum.repos.d"
#	mount -t iso9660 /dev/cdrom /media
#	stat "mount CDROM"
#	rm -rf  /etc/yum.repos.d/* && echo -e "[c7-media]\nname=CentOS-$releasever - Media\nbaseurl=file:///media/\ngpgcheck=0\nenabled=1" > /etc/yum.repos.d/centos-media.repo

#		yum clean all &>/dev/null
 #       yum makecache &>/dev/null
gzip /etc/yum.repos.d/*.repo && curl -o /etc/yum.repos.d/AnolisOS-BaseOS.repo http://192.168.3.100/yum/AnolisOS-BaseOS.repo  && curl -o /etc/yum.repos.d/AnolisOS-AppStream.repo  http://192.168.3.100/yum/AnolisOS-AppStream.repo && yum clean all  && yum makecache

         yum install -y vim-enhanced iproute netstat net-tools util-linux-ng gcc-c++ make cmake libxml2-devel openssl-devel \
                screen git mailx dos2unix lrzsz dstat xinetd rsync tree bind-utils  autoconf automake  \
                ntpdate curl wget zip unzip gcc man \
                dstat tcpdump telnet  expect lsof traceroute bind-utils --skip-broken   && stat "yum all packages"

}




ulimt_change() {
ulimit -SHn 65535
cat >> /etc/security/limits.conf <<EOF
root soft nofile 65535
root hard nofile 65535
* soft nproc 65535
* hard nproc 65535
* soft nofile 65535
* hard nofile 65535
EOF

cat >/etc/security/limits.d/90-nproc.conf<<EOF
*          soft    nproc     65535
root       soft    nproc     unlimited

EOF

echo "ulimit -SH 65535" >> /etc/rc.local

        ok "Ulimit: 65535"
stat "ulimit make"
}


optimize_ssh() {
sed -i 's/#UseDNS yes/UseDNS no/g' /etc/ssh/sshd_config &>/dev/null
        systemctl restart sshd &>/dev/null
sed -i '/^PASS_MAX_DAYS/c   PASS_MAX_DAYS   90'  /etc/login.defs
sed -i '/^PASS_MIN_DAYS/c   PASS_MIN_DAYS   2'   /etc/login.defs
sed -i '/^#PASS_MIN_LEN/c   PASS_MIN_LEN    8'   /etc/login.defs
sed -i '/^PASS_WARN_AGE/c   PASS_WARN_AGE   7'   /etc/login.defs


#sed -i '/PasswordAuthentication /c PasswordAuthentication no' /etc/ssh/sshd_config
systemctl restart sshd
stat "sshd config"
}


sysctl_optimize() {
sed -i 's/net.ipv4.tcp_syncookies.*$/net.ipv4.tcp_syncookies = 1/g' /etc/sysctl.conf &>/dev/null
cat >> /etc/sysctl.conf << ENDF
net.ipv4.tcp_max_syn_backlog = 65536
net.core.netdev_max_backlog =  32768
net.core.somaxconn = 32768
net.core.wmem_default = 8388608
net.core.rmem_default = 8388608
net.core.rmem_max = 16777216
net.core.wmem_max = 16777216
net.ipv4.tcp_timestamps = 0
net.ipv4.tcp_synack_retries = 2
net.ipv4.tcp_syn_retries = 2
net.ipv4.tcp_tw_recycle = 1
#net.ipv4.tcp_tw_len = 1
net.ipv4.tcp_tw_reuse = 1
net.ipv4.tcp_max_orphans = 3276800
net.ipv4.ip_local_port_range = 1024  65535
net.ipv4.ip_forward = 1
net.ipv4.conf.default.rp_filter = 0
net.ipv4.conf.all.rp_filter = 0
net.ipv4.conf.eth0.rp_filter = 0
net.ipv4.conf.default.accept_source_route = 0
kernel.sysrq = 0
kernel.core_uses_pid = 1
net.ipv4.tcp_syncookies = 1
kernel.msgmnb = 65536
kernel.msgmax = 65536
kernel.shmmax = 68719476736
kernel.shmall = 4294967296
net.ipv4.tcp_max_tw_buckets = 20000
net.ipv4.tcp_sack = 1
net.ipv4.tcp_window_scaling = 1
net.ipv4.tcp_rmem = 4096        87380   4194304
net.ipv4.tcp_wmem = 4096        16384   4194304
net.core.rmem_max = 16777216
net.core.wmem_max = 16777216
net.core.netdev_max_backlog = 262144
net.core.somaxconn = 65535
net.ipv4.tcp_max_syn_backlog = 262144
net.ipv4.tcp_timestamps = 1
net.ipv4.tcp_synack_retries = 1
net.ipv4.tcp_syn_retries = 3
net.ipv4.tcp_tw_recycle = 0
net.ipv4.tcp_mem = 94500000 915000000 927000000
net.ipv4.tcp_fin_timeout = 1
net.ipv4.tcp_keepalive_time = 300
net.ipv4.ip_local_port_range = 1024    65000
net.netfilter.nf_conntrack_max = 655360
net.netfilter.nf_conntrack_tcp_timeout_established = 1200

ENDF

lsmod |grep conntrack
modprobe ip_conntrack
sysctl -p  &>/dev/null && stat "add: sysctl"
}



firewalld() {
sed -i '/SELINUX/s/enforcing/disabled/' /etc/selinux/config && stat "selinux off"
setenforce 0
systemctl stop firewalld
stat "SElinux off"
stat "firewalld off"
systemctl disable firewalld

systemctl stop postfix
systemctl disable postfix
stat "firewalld disable"
}

make_history(){

# 系统安全配置 ####


cat > /tmp/ptywb.sh <<"PART"
# record user's operation script ####
PART

touch /var/log/useraudit.log
chmod 777 /var/log/useraudit.log
cat >>/etc/profile<<"EOF"
export TMOUT=600
export HISTTIMEFORMAT="%Y-%m-%d-%H:%M:%S   --- "
export HISTORY_FILE=/var/log/useraudit.log
export PROMPT_COMMAND='{ h=`history 1`;w=`who am i`;echo -e $(date "+%Y-%m-%d %H:%M:%S -- "  ) --- $w ---  $h;} >> $HISTORY_FILE'
EOF
echo "用户操作日志文件存放在/var/log/useraudit.log"
source /etc/profile
stat "history log"
}


delete_self(){
rm -f /opt/def.txt
rm -f `pwd`/$0
}

#status_check 123 true

optimize_ssh
sysctl_optimize
ulimt_change
make_history
firewalld
#password_make
#delete_self

内核升级

centos7默认内核为3.10.0-1160.el7.x86_64，问题较多，升级内核（5台都需要）；可以有效避秒cgroup内存泄露导入elrepo gpg keyrpm --import https://www.elrepo.org/RPM-GPG-KEY-elrepo.org安装elrepo YUM源仓库yum -y install https://www.elrepo.org/elrepo-release-7.0-4.el7.elrepo.noarch.rpm安装kernel-ml版本，ml为长期稳定版本，lt为长期维护版本yum --enablerepo="elrepo-kernel" -y install kernel-ml.x86_64
查看可装内核
yum --disablerepo="*" --enablerepo="elrepo-kernel" list available设置grub2默认引导为0grub2-set-default 0重新生成grub2引导文件grub2-mkconfig -o /boot/grub2/grub.cfg更新后，需要重启，使用升级的内核生效。reboot升级之后[root@test-new-k8s-master01 ~]# uname -r6.0.1-1.el7.elrepo.x86_64hishis

配置yum源：

cat kubernetes.repo
[kubernetes]
name=Kubernetes
baseurl=https://mirrors.aliyun.com/kubernetes-new/core/stable/v1.32/rpm/
enabled=1
gpgcheck=1
gpgkey=https://mirrors.aliyun.com/kubernetes-new/core/stable/v1.32/rpm/repodata/repomd.xml.key

----
 cat kylin_x86_64.repo
###Kylin Linux Advanced Server 10 - os repo###

[ks10-adv-os]
name = Kylin Linux Advanced Server 10 - Os
baseurl = http://update.cs2c.com.cn:8080/NS/V10/V10SP2/os/adv/lic/base/$basearch/
gpgcheck = 1
gpgkey=file:///etc/pki/rpm-gpg/RPM-GPG-KEY-kylin
enabled = 1

[ks10-adv-updates]
name = Kylin Linux Advanced Server 10 - Updates
baseurl = http://update.cs2c.com.cn:8080/NS/V10/V10SP2/os/adv/lic/updates/$basearch/
gpgcheck = 1
gpgkey=file:///etc/pki/rpm-gpg/RPM-GPG-KEY-kylin
enabled = 1

[ks10-adv-addons]
name = Kylin Linux Advanced Server 10 - Addons
baseurl = http://update.cs2c.com.cn:8080/NS/V10/V10SP2/os/adv/lic/addons/$basearch/
gpgcheck = 1
gpgkey=file:///etc/pki/rpm-gpg/RPM-GPG-KEY-kylin
enabled = 0

配置集群免密：
省略
安装docker

麒麟目前没有适配的,使用二进制安装
下载地址： https://download.docker.com/linux/static/stable/x86_64/  #根据需要下载对应版本，我这边下载的是28
tar xf docker-28.3.1.tgz
cp docker/
cp ./* /usr/local/bin/

~]# cat /etc/systemd/system/docker.service
[Unit]
Description=Docker Application Container Engine
Documentation=http://docs.docker.io
[Service]
OOMScoreAdjust=-1000
Environment="PATH=/usr/local/bin:/bin:/sbin:/usr/bin:/usr/sbin"
ExecStart=/usr/local/bin/dockerd
ExecStartPost=/sbin/iptables -I FORWARD -s 0.0.0.0/0 -j ACCEPT
ExecReload=/bin/kill -s HUP $MAINPID
Restart=on-failure
RestartSec=5
LimitNOFILE=infinity
LimitNPROC=infinity
LimitCORE=infinity
Delegate=yes
KillMode=process
[Install]
WantedBy=multi-user.target

----------
systemctl daemon-reload
systemctl start docker
systemctl enable docker

安装cri-docker

wget https://github.com/Mirantis/cri-dockerd/releases/download/v0.3.8/cri-dockerd-0.3.8-3.el7.x86_64.rpm

yum install containerd.io conntrack -y

rpm -ivh cri-dockerd-0.3.8-3.el7.x86_64.rpm
####这里要新增下不然，k8s集群初始化还是走Dec 11 15:52:48 kubesphere_master1 dockerd[10879]: time="2023-12-11T15:52:48.865021139+08:00" level=error msg="Handler for POST /v1.43/images/create returned error: Head \"https://us-west2-docker.pkg.dev/v2/k8s-artifacts-prod/images/pause/manifests/3.6\": dial tcp 74.125.23.82:443: connect: connection timed out"

vim /usr/lib/systemd/system/cri-docker.service
这里加上镜像地址，
ExecStart=/usr/bin/cri-dockerd --pod-infra-container-image=registry.cn-hangzhou.aliyuncs.com/google_containers/pause:3.9 --container-runtime-endpoint fd://

# 重载系统守护进程
sudo systemctl daemon-reload 
# 创建docker组，因为cri-docker依赖docker组，但是我们又是二进制安装的docker，没有这个组
sudo groupadd docker
# 设置cri-dockerd自启动
sudo systemctl enable cri-docker.socket cri-docker 
# 启动cri-dockerd
sudo systemctl start cri-docker.socket cri-docker 
# 查看cri-dockerd状态
sudo systemctl status cri-docker.socket

k8s基础配置：

关闭swap
swapoff -a

vim /etc/fstab
# 删除swap行

  # 加载系统模块
modprobe br_netfilter   
modprobe overlay lsmod | grep br_netfilter
lsmod | grep overlay

# 持久化配置：通过/etc/modules-load.d/目录
vim /etc/modules-load.d/k8s.conf
# 在文件中添加以下内容并保存
br_netfilter
overlay

systemctl restart systemd-modules-load


#添加网桥过滤及内核转发配置文件
cat > /etc/sysctl.d/k8s.conf << EOF
net.bridge.bridge-nf-call-ip6tables = 1
net.bridge.bridge-nf-call-iptables = 1
net.ipv4.ip_forward = 1
vm.swappiness = 0
EOF

#开启ipvs
cat > /etc/sysconfig/modules/ipvs.modules << EOF
#!/bin/sh
modprobe -- ip_vs
modprobe -- ip_vs_rr
modprobe -- ip_vs_wrr
modprobe -- ip_vs_sh
modprobe -- nf_conntrack
EOF

#生效配置文件
sysctl -p /etc/sysctl.d/k8s.conf
chmod 755 /etc/sysconfig/modules/ipvs.modules 
bash /etc/sysconfig/modules/ipvs.modules
lsmod | grep -e ip_vs -e nf_conntrack

安装k8s

#每个节点都安装
yum install -y kubelet kubeadm kubectl 
systemctl enable kubelet && systemctl start kubelet

8.配置kubeadm.yaml

]# cat kubeadm.yaml
apiVersion: kubeadm.k8s.io/v1beta4
bootstrapTokens:
- groups:
  - system:bootstrappers:kubeadm:default-node-token
  token: abcdef.0123456789abcdef
  ttl: 24h0m0s
  usages:
  - signing
  - authentication
kind: InitConfiguration
localAPIEndpoint:
  advertiseAddress: 172.16.26.121
  bindPort: 6443
nodeRegistration:
  criSocket: unix:///var/run/cri-dockerd.sock
  imagePullPolicy: IfNotPresent
  imagePullSerial: true
  name: k8s001
  taints: null
timeouts:
  controlPlaneComponentHealthCheck: 4m0s
  discovery: 5m0s
  etcdAPICall: 2m0s
  kubeletHealthCheck: 4m0s
  kubernetesAPICall: 1m0s
  tlsBootstrap: 5m0s
  upgradeManifests: 5m0s
---
apiServer: {}
apiVersion: kubeadm.k8s.io/v1beta4
caCertificateValidityPeriod: 87600h0m0s
certificateValidityPeriod: 8760h0m0s
certificatesDir: /etc/kubernetes/pki
controlPlaneEndpoint: "172.16.26.124:6443"
clusterName: kubernetes
controllerManager: {}
dns: {}
encryptionAlgorithm: RSA-2048
etcd:
  local:
    dataDir: /var/lib/etcd
imageRepository: registry.cn-hangzhou.aliyuncs.com/google_containers
kind: ClusterConfiguration
kubernetesVersion: 1.32.0
networking:
  dnsDomain: cluster.local
  podSubnet: 10.222.0.0/16
  serviceSubnet: 10.96.0.0/12
proxy: {}
scheduler: {}
---
apiVersion: kubeproxy.config.k8s.io/v1alpha1
kind: KubeProxyConfiguration
mode: ipvs
---
apiVersion: kubelet.config.k8s.io/v1beta1
kind: KubeletConfiguration
cgroupDriver: systemd

# 其中172.16.26.124:6443 是我用nginx做的4层代理，用来master负载， 172.16.26.121 是本机地址

初始化：

kubeadm init --config=kubeadm.yaml --upload-certs

-------