安装Ansible
yum install -y https://dl.fedoraproject.org/pub/epel/epel-release-latest-7.noarch.rpm #添加repo源
yum install -y ansible.noarch #安装ansible配置ansible的hosts文件
cat >/etc/ansible/hosts <<EOF
[master]
master1 ansible_ssh_host="192.168.137.100"
master2 ansible_ssh_host="192.168.137.110"
master3 ansible_ssh_host="192.168.137.120"
EOFssh-keygen -t rsa #生成密钥
Generating public/private rsa key pair.
Enter file in which to save the key (/root/.ssh/id_rsa):
Created directory '/root/.ssh'.
Enter passphrase (empty for no passphrase):
Enter same passphrase again:
Your identification has been saved in /root/.ssh/id_rsa.
Your public key has been saved in /root/.ssh/id_rsa.pub.
The key fingerprint is:
SHA256:2iC7vtK7x1XEdXhvQMhHTAJ/4XfP1WyFGfID5siGKA0 root@localhost.localdomain
The key's randomart image is:
+---[RSA 2048]----+
| E ..+*BB+.|
| o . oo=+BBoo|
| . o ..+ oo=oB|
| . .. . =*|
| . . S. .o|
| o +. |
| ...... |
| . ..o |
| o*= |
+----[SHA256]-----+发送公钥到各个节点，实现免密码登陆
ssh-copy-id -i /root/.ssh/id_rsa.pub 192.168.137.100:
ssh-copy-id -i /root/.ssh/id_rsa.pub 192.168.137.110:
ssh-copy-id -i /root/.ssh/id_rsa.pub 192.168.137.120:设置主机名
hostnamectl set-hostname master1
ssh 192.168.137.110 "hostnamectl set-hostname master2"
ssh 192.168.137.120 "hostnamectl set-hostname master3"添加hosts文件
ansible all -m shell -a "echo '192.168.137.100 master1' >> /etc/hosts"
ansible all -m shell -a "echo '192.168.137.110 master2' >> /etc/hosts"
ansible all -m shell -a "echo '192.168.137.120 master3' >> /etc/hosts"
ansible all -m shell -a "echo '192.168.137.200 vip' >> /etc/hosts"关闭selinux和防火墙
ansible all -m shell -a "setenforce 0 &&systemctl --now disable firewalld && sed -i 's/SELINUX=enforcing/SELINUX=disabled/g' /etc/selinux/config"安装依赖文件
ansible all -m shell -a "yum install yum-utils device-mapper-persistent-data lvm2 vim wget yum-utils lrzsz lsof ipvsadm git net-tools libseccomp ipset jq iptables sysstat chrony -y "
ansible all -m shell -a "yum -y install iptables-services && systemctl start iptables && systemctl enable iptables && iptables -F && service iptables save" #iptables策略放空
ansible all -m shell -a "systemctl stop postfix.service && systemctl disable postfix.service"安装chrony，并开启，实现时间同步
ansible all -m shell -a "yum install -y chrony && systemctl --now enable chronyd"查看时间同步
ansible all -m shell -a "timedatectl"
master3 | CHANGED | rc=0 >>
Local time: Thu 2020-09-10 20:10:47 CST
Universal time: Thu 2020-09-10 12:10:47 UTC
RTC time: Thu 2020-09-10 12:10:47
Time zone: Asia/Shanghai (CST, +0800)
NTP enabled: yes
NTP synchronized: yes
RTC in local TZ: no### 修改K8S内核参数
cat <<EOF > /etc/sysctl.d/k8s.conf
net.bridge.bridge-nf-call-iptables=1
net.bridge.bridge-nf-call-ip6tables=1
net.ipv4.ip_forward=1
net.ipv4.tcp_tw_recycle=0
vm.swappiness=0
vm.overcommit_memory=1
vm.panic_on_oom=0
fs.inotify.max_user_instances=8192
fs.inotify.max_user_watches=1048576
fs.file-max=52706963
fs.nr_open=52706963
net.ipv6.conf.all.disable_ipv6=1net.netfilter.nf_conntrack_max=2310720
vm.max_map_count=262144
EOF复制到其他主机
ansible all -m copy -a "src=/etc/sysctl.d/k8s.conf dest=/etc/sysctl.d/k8s.conf"
ansible all -m shell -a "sysctl -p /etc/sysctl.d/k8s.conf"设置 rsyslogd 和 systemd journald
ansible all -m shell -a "mkdir /var/log/journal"
ansible all -m shell -a "mkdir /etc/systemd/journald.conf.d"
cat > /etc/systemd/journald.conf.d/99-prophet.conf <<EOF
[Journal] # 持久化保存到磁盘
Storage=persistent # 压缩历史日志
Compress=yes
SyncIntervalSec=5m
RateLimitInterval=30s
RateLimitBurst=1000 #最大占用空间
10GSystemMaxUse=10G #单日志文件最大
200MSystemMaxFileSize=200M #日志保存时间 2 周
MaxRetentionSec=2week #不将日志转发到
syslogForwardToSyslog=no
EOFansible all -m copy -a "src=/etc/systemd/journald.conf.d/99-prophet.conf dest=/etc/systemd/journald.conf.d/99-prophet.conf"
ansible all -m shell -a "systemctl restart systemd-journald"### 升级内核
ansible all -m shell -a "rpm -import https://www.elrepo.org/RPM-GPG-KEY-elrepo.org" #导入key
ansible all -m shell -a "rpm -Uvh http://www.elrepo.org/elrepo-release-7.0-2.el7.elrepo.noarch.rpm" #安装elrepo源
yum --diablerepo="*" --enablerepo="elrepo-kernel" list available #查看可用内核包
ansible all -m shell -a "yum -y --enablerepo=elrepo-kernel install kernel-ml.x86_64 kernel-ml-devel.x86_64 “ #安装最新内核awk -F\' '$1=="menuentry " {print $2}' /etc/grub2.cfg #查看新内核顺序为0
vim /etc/default/grub
GRUB_TIMEOUT=5
GRUB_DISTRIBUTOR="$(sed 's, release .*$,,g' /etc/system-release)"
GRUB_DEFAULT=0 #修改为0
GRUB_DISABLE_SUBMENU=true
GRUB_TERMINAL_OUTPUT="console"
GRUB_CMDLINE_LINUX="crashkernel=auto rd.lvm.lv=centos/root rd.lvm.lv=centos/swap rhgb quiet numa=off" #末尾加numa=off ，关闭numa
GRUB_DISABLE_RECOVERY="true"ansible all -m shell -a "cp /boot/grub2/grub.cfg{,.bak}" #备份grub.cfg文件
ansible all -m shell -a "grub2-mkconfig -o /boot/grub2/grub.cfg" #创建内核配置
ansible all -m shell -a "reboot " #重启电脑
ansible all -m shell -a "uname -a " #查看内核
Linux master2 5.8.8-1.el7.elrepo.x86_64 #1 SMP Wed Sep 9 14:47:37 EDT 2020 x86_64 x86_64 x86_64 GNU/Linuxcat <<EOF > /etc/yum.repos.d/kubernetes.repo
[kubernetes]
name=Kubernetes
baseurl=https://mirrors.aliyun.com/kubernetes/yum/repos/kubernetes-el7-x86_64
enabled=1
gpgcheck=0
repo_gpgcheck=0
gpgkey=https://mirrors.aliyun.com/kubernetes/yum/doc/yum-key.gpg
http://mirrors.aliyun.com/kubernetes/yum/doc/rpm-package-key.gpg
EOFcurl -o /etc/yum.repos.d/docker-ce.repo https://mirrors.aliyun.com/docker-ce/linux/centos/docker-ce.repocurl -o /tmp/rpm-package-key.gpg https://mirrors.aliyun.com/kubernetes/yum/doc/rpm-package-key.gpg
ansible all -m copy -a "src=/tmp/rpm-package-key.gpg dest=/tmp/rpm-package-key.gpg"
ansible all -m copy -a "src=/etc/yum.repos.d/docker-ce.repo dest=/etc/yum.repos.d/docker-ce.repo"
ansible all -m copy -a "src=/etc/yum.repos.d/kubernetes.repo dest=/etc/yum.repos.d/kubernetes.repo"yum update -y #每台都执行
yum install -y docker-ce #每台都执行
ansible all -m shell -a "systemctl --now enable docker"
ansible all -m shell -a "mkdir -p /etc/docker &&mkdir -p /etc/systemd/system/docker.service.d"
cat > /etc/docker/daemon.json <<EOF
{
"exec-opts": ["native.cgroupdriver=systemd"],
"log-driver": "json-file",
"log-opts": {
"max-size": "100m"
},
"storage-driver": "overlay2",
"storage-opts": [
"overlay2.override_kernel_check=true"
],
"registry-mirrors": [
"https://v2ltjwbg.mirror.aliyuncs.com",
"https://docker.mirrors.ustc.edu.cn",
"http://f1361db2.m.daocloud.io",
"https://registry.docker-cn.com"
]
}
EOFmodprobe br_netfilter
cat > /etc/sysconfig/modules/ipvs.modules <<EOF
#!/bin/bash
modprobe -- ip_vs
modprobe -- ip_vs_rr
modprobe -- ip_vs_wrr
modprobe -- ip_vs_sh
modprobe -- nf_conntrack_ipv4
EOF
chmod 755 /etc/sysconfig/modules/ipvs.modules && bash
/etc/sysconfig/modules/ipvs.modules && lsmod |grep -e ip_vs -e nf_conntrack_ipv4yum install -y kubectl kubeadm kubelet #每台上都执行
ansible all -m shell -a "systemctl enable kubelet"获取所需要镜像链接
[root@master1 ~]# kubeadm config images list
W0917 19:42:42.158905 1700 configset.go:348] WARNING: kubeadm cannot validate component configs for API groups [kubelet.config.k8s.io kubeproxy.config.k8s.io]
k8s.gcr.io/kube-apiserver:v1.19.2
k8s.gcr.io/kube-controller-manager:v1.19.2
k8s.gcr.io/kube-scheduler:v1.19.2
k8s.gcr.io/kube-proxy:v1.19.2
k8s.gcr.io/pause:3.2
k8s.gcr.io/etcd:3.4.13-0
k8s.gcr.io/coredns:1.7.0### 因不能访问外网，可先上传到镜像到Docker Hub，再从Docker Hub上进行拉取，详情请看如果从Docker Hub拉取镜像：
[root@master1 ~]# docker login
Login with your Docker ID to push and pull images from Docker Hub. If you don't have a Docker ID, head over to https://hub.docker.com to create one.
Username: linben1985
Password:
WARNING! Your password will be stored unencrypted in /root/.docker/config.json.
Configure a credential helper to remove this warning. See
https://docs.docker.com/engine/reference/commandline/login/#credentials-store
Login Succeeded
docker拉取镜像地址，在docker hub上复制即可，所需要的全都拉取下来
[root@master1 ~]# docker pull linben1985/k8sofben:v1.19.2
v1.19.2: Pulling from linben1985/k8sofben
b9cd0ea6c874: Pull complete
a84ff2cd01b7: Pull complete
f5db63e1da64: Pull complete
Digest: sha256:b119baef2a60b537c264c0ea009f63095169af089e1a36fb4167693f1b60cd1e
Status: Downloaded newer image for linben1985/k8sofben:v1.19.2
docker.io/linben1985/k8sofben:v1.19.2
[root@master1 ~]# docker images
REPOSITORY TAG IMAGE ID CREATED SIZE
linben1985/k8sofben v1.19.2 607331163122 24 hours ago 119MB
wise2c/keepalived-k8s latest 0ba6a7862982 2 years ago 14MB
wise2c/haproxy-k8s latest fde31577093d 2 years ago 71.1MB
把拉取的镜像tag设置成kubeadm config images list获取的地址，因为获取的是默认的地址，后面初始化时不用修改镜像地址
[root@master1 ~]# docker tag linben1985/k8sofben:v1.19.2 k8s.gcr.io/kube-apiserver:v1.19.2
[root@master1 ~]# docker images
REPOSITORY TAG IMAGE ID CREATED SIZE
linben1985/k8sofben v1.19.2 607331163122 24 hours ago 119MB
k8s.gcr.io/kube-apiserver v1.19.2 607331163122 24 hours ago 119MB
wise2c/keepalived-k8s latest 0ba6a7862982 2 years ago 14MB
wise2c/haproxy-k8s latest fde31577093d 2 years ago 71.1MB
[root@master1 ~]# docker rmi linben1985/k8sofben:v1.19.2
Untagged: linben1985/k8sofben:v1.19.2
Untagged: linben1985/k8sofben@sha256:b119baef2a60b537c264c0ea009f63095169af089e1a36fb4167693f1b60cd1e
[root@master1 ~]# docker images
REPOSITORY TAG IMAGE ID CREATED SIZE
k8s.gcr.io/kube-apiserver v1.19.2 607331163122 24 hours ago 119MB
wise2c/keepalived-k8s latest 0ba6a7862982 2 years ago 14MB
wise2c/haproxy-k8s latest fde31577093d 2 years ago 71.1MB
docker images
REPOSITORY TAG IMAGE ID CREATED SIZE
linben1985/k8sofben proxy d373dd5a8593 47 hours ago 118MB
linben1985/k8sofben controller 8603821e1a7a 47 hours ago 111MB
k8s.gcr.io/kube-apiserver v1.19.2 607331163122 47 hours ago 119MB
linben1985/k8sofben scheduler 2f32d66b884f 47 hours ago 45.7MB
linben1985/k8sofben etcd 0369cf4303ff 3 weeks ago 253MB
linben1985/k8sofben coredns bfe3a36ebd25 3 months ago 45.2MB
linben1985/k8sofben pause 80d28bedfe5d 7 months ago 683kB
wise2c/keepalived-k8s latest 0ba6a7862982 2 years ago 14MB
wise2c/haproxy-k8s latest fde31577093d 2 years ago 71.1MB
[root@master1 ~]# kubeadm config images list
W0918 22:58:50.017625 1835 configset.go:348] WARNING: kubeadm cannot validate component configs for API groups [kubelet.config.k8s.io kubeproxy.config.k8s.io]
k8s.gcr.io/kube-apiserver:v1.19.2
k8s.gcr.io/kube-controller-manager:v1.19.2
k8s.gcr.io/kube-scheduler:v1.19.2
k8s.gcr.io/kube-proxy:v1.19.2
k8s.gcr.io/pause:3.2
k8s.gcr.io/etcd:3.4.13-0
k8s.gcr.io/coredns:1.7.0
[root@master1 ~]# docker tag d373dd5a8593 k8s.gcr.io/kube-proxy:v1.19.2
[root@master1 ~]# docker tag 8603821e1a7a k8s.gcr.io/kube-controller-manager:v1.19.2
[root@master1 ~]# docker tag 2f32d66b884f k8s.gcr.io/kube-scheduler:v1.19.2
[root@master1 ~]# docker tag 0369cf4303ff k8s.gcr.io/etcd:3.4.13-0
[root@master1 ~]# docker tag bfe3a36ebd25 k8s.gcr.io/coredns:1.7.0
[root@master1 ~]# docker tag 80d28bedfe5d k8s.gcr.io/pause:3.2
把修改tag后的镜像保存到本地，并传给其他两个节点
mkdir images && cd images
root@master1 images]# docker save -o api.tar k8s.gcr.io/kube-apiserver:v1.19.2
[root@master1 images]# docker save -o controller.tar k8s.gcr.io/kube-controller-manager:v1.19.2
[root@master1 images]# docker save -o scheduler.tar k8s.gcr.io/kube-scheduler:v1.19.2
[root@master1 images]# docker save -o proxy.tar k8s.gcr.io/kube-proxy:v1.19.2
[root@master1 images]# docker save -o pause.tar k8s.gcr.io/pause:3.2
[root@master1 images]# docker save -o etcd.tar k8s.gcr.io/etcd:3.4.13-0
[root@master1 images]# docker save -o coredns.tar k8s.gcr.io/coredns:1.7.0
[root@master1 images]# ls
api.tar controller.tar coredns.tar etcd.tar pause.tar proxy.tar scheduler.tar
[root@master1 k8s]# scp -r images/ root@192.168.137.110:/root
[root@master1 k8s]# scp -r images/ root@192.168.137.120:/root
master2和master3上操作
[root@master2 images]# vim load-images.sh
#!/bin/bash
cd /root/images/ #修改为镜像路径
ls /root/images/ | grep -v load-images.sh > /tmp/k8s-images.txt #修改为镜像路径
for i in $( cat /tmp/k8s-images.txt )
do
docker load -i $i
done
rm -rf /tmp/k8s-images.txt[root@master2 images]# sh load-images.sh
[root@master2 images]# docker images
REPOSITORY TAG IMAGE ID CREATED SIZE
k8s.gcr.io/kube-proxy v1.19.2 d373dd5a8593 2 days ago 118MB
k8s.gcr.io/kube-apiserver v1.19.2 607331163122 2 days ago 119MB
k8s.gcr.io/kube-controller-manager v1.19.2 8603821e1a7a 2 days ago 111MB
k8s.gcr.io/kube-scheduler v1.19.2 2f32d66b884f 2 days ago 45.7MB
k8s.gcr.io/etcd 3.4.13-0 0369cf4303ff 3 weeks ago 253MB
k8s.gcr.io/coredns 1.7.0 bfe3a36ebd25 3 months ago 45.2MB
k8s.gcr.io/pause 3.2 80d28bedfe5d 7 months ago 683kB
[root@master1 etc]# vi haproxy.cfg
global
log 127.0.0.1 local0
log 127.0.0.1 local1 notice
maxconn 4096
#chroot /usr/share/haproxy
#user haproxy
#group haproxy
daemon
defaults
log global
mode http
option httplog
option dontlognull
retries 3
option redispatch
timeout connect 5000
timeout client 50000
timeout server 50000
frontend stats-front
bind *:8081
mode http
default_backend stats-back
frontend fe_k8s_6444
bind *:6444
mode tcp
timeout client 1h
log global
option tcplog
default_backend be_k8s_6443
acl is_websocket hdr(Upgrade) -i WebSocket
acl is_websocket hdr_beg(Host) -i ws
backend stats-back
mode http
balance roundrobin
stats uri /haproxy/stats
stats auth pxcstats:secret
backend be_k8s_6443
mode tcp
timeout queue 1h
timeout server 1h
timeout connect 1h
log global
balance roundrobin
server rancher01 192.168.137.100:6443 #指定三台master的IP
server rancher02 192.168.137.110:6443
server rancher03 192.168.137.120:6443
[root@master1 lb]# vi start-haproxy.sh
#!/bin/bash
MasterIP1=192.168.137.100 #指定三台master的IP
MasterIP2=192.168.137.110
MasterIP3=192.168.137.120
MasterPort=6443
docker run -d --restart=always --name HAProxy-K8S -p 6444:6444 \
-e MasterIP1=$MasterIP1 \
-e MasterIP2=$MasterIP2 \
-e MasterIP3=$MasterIP3 \
-e MasterPort=$MasterPort \
-v /data/lb/etc/haproxy.cfg:/usr/local/etc/haproxy/haproxy.cfg \
wise2c/haproxy-k8s #/data/lb/etc/haproxy.cfg路径为上面生成的路径
[root@master1 lb]# vi start-keepalived.sh
#!/bin/bash
VIRTUAL_IP=192.168.137.200 #指定VIP的IP
INTERFACE=ens33 #指定现用的网络接口
NETMASK_BIT=24
CHECK_PORT=6444
RID=10
VRID=160
MCAST_GROUP=224.0.0.18
docker run -itd --restart=always --name=Keepalived-K8S \
--net=host --cap-add=NET_ADMIN \
-e VIRTUAL_IP=$VIRTUAL_IP \
-e INTERFACE=$INTERFACE \
-e CHECK_PORT=$CHECK_PORT \
-e RID=$RID \
-e VRID=$VRID \
-e NETMASK_BIT=$NETMASK_BIT \
-e MCAST_GROUP=$MCAST_GROUP \
wise2c/keepalived-k8s[root@master1 lb]# sh start-haproxy.sh
[root@master1 lb]# netstat -anp|grep proxy
tcp6 0 0 :::6444 :::* LISTEN 2098/docker-proxy
[root@master1 lb]# sh start-keepalived.sh
[root@master1 lb]# ip add
1: lo: <LOOPBACK,UP,LOWER_UP> mtu 65536 qdisc noqueue state UNKNOWN group default qlen 1000
link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00
inet 127.0.0.1/8 scope host lo
valid_lft forever preferred_lft forever
inet6 ::1/128 scope host
valid_lft forever preferred_lft forever
2: ens33: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc pfifo_fast state UP group default qlen 1000
link/ether 00:0c:29:46:1f:63 brd ff:ff:ff:ff:ff:ff
inet 192.168.137.100/24 brd 192.168.137.255 scope global noprefixroute ens33
valid_lft forever preferred_lft forever
inet 192.168.137.200/24 scope global secondary ens33 #VIP IP
valid_lft forever preferred_lft forever
inet6 fe80::c8c1:ce38:9736:7440/64 scope link noprefixroute
valid_lft forever preferred_lft forever
[root@master1 lb]# docker ps
CONTAINER ID IMAGE COMMAND CREATED STATUS PORTS NAMES
0bf63e8f9e7e wise2c/keepalived-k8s "/usr/bin/keepalived…" About a minute ago Up About a minute Keepalived-K8S
1de5062184ea wise2c/haproxy-k8s "/docker-entrypoint.…" 3 minutes ago Up 3 minutes 0.0.0.0:6444->6444/tcp HAProxy-K8S
kubeadm config print init-defaults > kubeadm-config.yaml #生成初始化文件
[root@master1 k8s]# vi kubeadm-config.yaml
apiVersion: kubeadm.k8s.io/v1beta2
bootstrapTokens:
- groups:
- system:bootstrappers:kubeadm:default-node-token
token: abcdef.0123456789abcdef
ttl: 24h0m0s
usages:
- signing
- authentication
kind: InitConfiguration
localAPIEndpoint:
advertiseAddress: 192.168.137.100 #修改为本机IP
bindPort: 6443
nodeRegistration:
criSocket: /var/run/dockershim.sock
name: master1
taints:
- effect: NoSchedule
key: node-role.kubernetes.io/master
---
apiServer:
timeoutForControlPlane: 4m0s
apiVersion: kubeadm.k8s.io/v1beta2
certificatesDir: /etc/kubernetes/pki
clusterName: kubernetes
controlPlaneEndpoint: "192.168.137.200:6444" #增加这一行，添加VIP ip
controllerManager: {}
dns:
type: CoreDNS
etcd:
local:
dataDir: /var/lib/etcd
imageRepository: k8s.gcr.io
kind: ClusterConfiguration
kubernetesVersion: v1.19.0
networking:
dnsDomain: cluster.local
serviceSubnet: 10.96.0.0/12
podSubnet: "10.244.0.0/16" #增加这一行，添加IPVS的IP段
scheduler: {}
--- #增加下面内容，设置为IPVS模式
apiVersion: kubeproxy.config.k8s.io/v1alpha1
kind: KubeProxyConfiguration
featureGates:
SupportIPVSProxyMode: true
mode: ipvs
[root@master1 k8s]# kubeadm init --config=kubeadm-config.yaml --upload-certs --ignore-preflight-errors=all| tee kubeadm-init.log #初始化主节点，第一次不用--ignore-preflight-errors=all 参数，如果第一次初始化失败，要修改某些参数的话，再初始化要使用这参数W0919 21:00:30.096513 110409 configset.go:348] WARNING: kubeadm cannot validate component configs for API groups [kubelet.config.k8s.io kubeproxy.config.k8s.io]
[init] Using Kubernetes version: v1.19.2
[preflight] Running pre-flight checks
[WARNING Port-10259]: Port 10259 is in use
[WARNING Port-10257]: Port 10257 is in use
[WARNING FileAvailable--etc-kubernetes-manifests-kube-apiserver.yaml]: /etc/kubernetes/manifests/kube-apiserver.yaml already exists
[WARNING FileAvailable--etc-kubernetes-manifests-kube-controller-manager.yaml]: /etc/kubernetes/manifests/kube-controller-manager.yaml already exists
[WARNING FileAvailable--etc-kubernetes-manifests-kube-scheduler.yaml]: /etc/kubernetes/manifests/kube-scheduler.yaml already exists
[WARNING FileAvailable--etc-kubernetes-manifests-etcd.yaml]: /etc/kubernetes/manifests/etcd.yaml already exists
[WARNING Port-10250]: Port 10250 is in use
[preflight] Pulling images required for setting up a Kubernetes cluster
[preflight] This might take a minute or two, depending on the speed of your internet connection
[preflight] You can also perform this action in beforehand using 'kubeadm config images pull'
[certs] Using certificateDir folder "/etc/kubernetes/pki"
[certs] Using existing ca certificate authority
[certs] Using existing apiserver certificate and key on disk
[certs] Using existing apiserver-kubelet-client certificate and key on disk
[certs] Using existing front-proxy-ca certificate authority
[certs] Using existing front-proxy-client certificate and key on disk
[certs] Using existing etcd/ca certificate authority
[certs] Using existing etcd/server certificate and key on disk
[certs] Using existing etcd/peer certificate and key on disk
[certs] Using existing etcd/healthcheck-client certificate and key on disk
[certs] Using existing apiserver-etcd-client certificate and key on disk
[certs] Using the existing "sa" key
[kubeconfig] Using kubeconfig folder "/etc/kubernetes"
[endpoint] WARNING: port specified in controlPlaneEndpoint overrides bindPort in the controlplane address
[kubeconfig] Using existing kubeconfig file: "/etc/kubernetes/admin.conf"
[endpoint] WARNING: port specified in controlPlaneEndpoint overrides bindPort in the controlplane address
[kubeconfig] Using existing kubeconfig file: "/etc/kubernetes/kubelet.conf"
[endpoint] WARNING: port specified in controlPlaneEndpoint overrides bindPort in the controlplane address
[kubeconfig] Using existing kubeconfig file: "/etc/kubernetes/controller-manager.conf"
[endpoint] WARNING: port specified in controlPlaneEndpoint overrides bindPort in the controlplane address
[kubeconfig] Using existing kubeconfig file: "/etc/kubernetes/scheduler.conf"
[kubelet-start] Writing kubelet environment file with flags to file "/var/lib/kubelet/kubeadm-flags.env"
[kubelet-start] Writing kubelet configuration to file "/var/lib/kubelet/config.yaml"
[kubelet-start] Starting the kubelet
[control-plane] Using manifest folder "/etc/kubernetes/manifests"
[control-plane] Creating static Pod manifest for "kube-apiserver"
[control-plane] Creating static Pod manifest for "kube-controller-manager"
[control-plane] Creating static Pod manifest for "kube-scheduler"
[etcd] Creating static Pod manifest for local etcd in "/etc/kubernetes/manifests"
[wait-control-plane] Waiting for the kubelet to boot up the control plane as static Pods from directory "/etc/kubernetes/manifests". This can take up to 4m0s
[apiclient] All control plane components are healthy after 20.503460 seconds
[upload-config] Storing the configuration used in ConfigMap "kubeadm-config" in the "kube-system" Namespace
[kubelet] Creating a ConfigMap "kubelet-config-1.19" in namespace kube-system with the configuration for the kubelets in the cluster
[upload-certs] Storing the certificates in Secret "kubeadm-certs" in the "kube-system" Namespace
[upload-certs] Using certificate key:
a79e48f9a158b8bad7a55672c8d21bef7307aedb10fb9f003739f2e7b5abc41d
[mark-control-plane] Marking the node master1 as control-plane by adding the label "node-role.kubernetes.io/master=''"
[mark-control-plane] Marking the node master1 as control-plane by adding the taints [node-role.kubernetes.io/master:NoSchedule]
[bootstrap-token] Using token: abcdef.0123456789abcdef
[bootstrap-token] Configuring bootstrap tokens, cluster-info ConfigMap, RBAC Roles
[bootstrap-token] configured RBAC rules to allow Node Bootstrap tokens to get nodes
[bootstrap-token] configured RBAC rules to allow Node Bootstrap tokens to post CSRs in order for nodes to get long term certificate credentials
[bootstrap-token] configured RBAC rules to allow the csrapprover controller automatically approve CSRs from a Node Bootstrap Token
[bootstrap-token] configured RBAC rules to allow certificate rotation for all node client certificates in the cluster
[bootstrap-token] Creating the "cluster-info" ConfigMap in the "kube-public" namespace
[kubelet-finalize] Updating "/etc/kubernetes/kubelet.conf" to point to a rotatable kubelet client certificate and key
[addons] Applied essential addon: CoreDNS
[endpoint] WARNING: port specified in controlPlaneEndpoint overrides bindPort in the controlplane address
[addons] Applied essential addon: kube-proxy
Your Kubernetes control-plane has initialized successfully!
To start using your cluster, you need to run the following as a regular user:
mkdir -p $HOME/.kube #以下三行为增加kubectl的admin权限
sudo cp -i /etc/kubernetes/admin.conf $HOME/.kube/config
sudo chown $(id -u):$(id -g) $HOME/.kube/config
You should now deploy a pod network to the cluster.
Run "kubectl apply -f [podnetwork].yaml" with one of the options listed at:
https://kubernetes.io/docs/concepts/cluster-administration/addons/
You can now join any number of the control-plane node running the following command on each as root:
kubeadm join 192.168.137.200:6444 --token abcdef.0123456789abcdef \
--discovery-token-ca-cert-hash sha256:e8da3dfb6d186081f2a42dcf078d880a6a364e1d89e3c51434be602b124d9941 \
--control-plane --certificate-key a79e48f9a158b8bad7a55672c8d21bef7307aedb10fb9f003739f2e7b5abc41d #master节点加入命令
Please note that the certificate-key gives access to cluster sensitive data, keep it secret!
As a safeguard, uploaded-certs will be deleted in two hours; If necessary, you can use
"kubeadm init phase upload-certs --upload-certs" to reload certs afterward.
Then you can join any number of worker nodes by running the following on each as root:
kubeadm join 192.168.137.200:6444 --token abcdef.0123456789abcdef \
--discovery-token-ca-cert-hash sha256:e8da3dfb6d186081f2a42dcf078d880a6a364e1d89e3c51434be602b124d9941 #普通Node加入命令
master2和master3上执行，加入master节点
kubeadm join 192.168.137.200:6444 --token abcdef.0123456789abcdef \
--discovery-token-ca-cert-hash sha256:96d2cfbcc77d10550aa05b98c092bc15be9693d784aa9ed48e4a4b6c20a777c6 \
--control-plane --certificate-key 41b6cc1edd842af85f9ce7186dd00af3f1c5f5e1db954a4d33381b65e511b6b1
所有节点都执行，增加kubectl的admin权限
mkdir -p $HOME/.kube
sudo cp -i /etc/kubernetes/admin.conf $HOME/.kube/config
sudo chown $(id -u):$(id -g) $HOME/.kube/config
[root@master1 k8s]# kubectl get node
NAME STATUS ROLES AGE VERSION
master1 NotReady master 8m34s v1.19.1
master2 NotReady master 7m30s v1.19.1
master3 NotReady master 3m56s v1.19.1
可以看到的有组件都有三个副本
[root@master1 k8s]# kubectl get pod --all-namespaces
NAMESPACE NAME READY STATUS RESTARTS AGE
kube-system coredns-f9fd979d6-fscvz 0/1 Pending 0 8m56s
kube-system coredns-f9fd979d6-rprfl 0/1 Pending 0 8m56s
kube-system etcd-master1 1/1 Running 0 9m4s
kube-system etcd-master2 1/1 Running 0 8m8s
kube-system etcd-master3 1/1 Running 0 4m34s
kube-system kube-apiserver-master1 1/1 Running 0 9m4s
kube-system kube-apiserver-master2 1/1 Running 0 8m8s
kube-system kube-apiserver-master3 1/1 Running 0 4m34s
kube-system kube-controller-manager-master1 1/1 Running 1 9m4s
kube-system kube-controller-manager-master2 1/1 Running 0 8m8s
kube-system kube-controller-manager-master3 1/1 Running 0 4m34s
kube-system kube-proxy-8tnrp 1/1 Running 0 4m36s
kube-system kube-proxy-dktgb 1/1 Running 0 8m10s
kube-system kube-proxy-jkxfj 1/1 Running 0 8m55s
kube-system kube-scheduler-master1 1/1 Running 1 9m4s
kube-system kube-scheduler-master2 1/1 Running 0 8m9s
kube-system kube-scheduler-master3 1/1 Running 0 4m34s设置按tab键补全kubectl命令
ansible all -m shell -a "source /usr/share/bash-completion/bash_completion"
ansible all -m shell -a “echo ‘source /usr/share/bash-completion/bash_completion’ >> ~/.bashrc”
ansible all -m shell -a ”echo ‘source <(kubectl completion bash)’ >> ~/.bashrc“部署flannel网络
kubectl apply -f https://raw.githubusercontent.com/coreos/flannel/master/Documentation/kube-flannel.yml
podsecuritypolicy.policy/psp.flannel.unprivileged created
clusterrole.rbac.authorization.k8s.io/flannel created
clusterrolebinding.rbac.authorization.k8s.io/flannel created
serviceaccount/flannel created
configmap/kube-flannel-cfg created
daemonset.apps/kube-flannel-ds created
[root@master1 manifests]# kubectl get node #查看节点状态，已经都是ready状态
NAME STATUS ROLES AGE VERSION
master1 Ready master 64d v1.19.1
master2 Ready master 64d v1.19.1
master3 Ready master 64d v1.19.1
