实验手工签发etcd集群证书，容器部署etcd集群

1、下载证书签发工具

wget https://pkg.cfssl.org/R1.2/cfssl_linux-amd64
wget https://pkg.cfssl.org/R1.2/cfssljson_linux-amd64
wget https://pkg.cfssl.org/R1.2/cfssl-certinfo_linux-amd64
chmod +x cfssl_linux-amd64 cfssljson_linux-amd64 cfssl-certinfo_linux-amd64
cp cfssl_linux-amd64 /usr/local/bin/cfssl
cp cfssljson_linux-amd64 /usr/local/bin/cfssljson
cp cfssl-certinfo_linux-amd64 /usr/bin/cfssl-certinfo

2、签发etcd集群证书

1、登录etcd1服务器，创建目录
mkdir -p  /data/etcd/{certs,data}
cd /root/kubernetes/certjson/

2、上传文件etcd-ca-config.json、etcd-ca-csr.json、etcd-server-csr.json、etcd-peer-csr.json 、etcd-client-csr.json到目录/root/kubernetes/certjson/
# 配置签发证书的期限为100年

3、签发etcd CA证书
cfssl gencert -initca /root/kubernetes/certjson/etcd-ca-csr.json | cfssljson -bare /data/etcd/certs/etcd-ca
# 校验etcd CA证书期限
openssl x509 -in /data/etcd/certs/etcd-ca.pem -text -noout | grep Not 

4、签发etcd server证书
cfssl gencert -ca=/data/etcd/certs/etcd-ca.pem -ca-key=/data/etcd/certs/etcd-ca-key.pem -config=/root/kubernetes/certjson/etcd-ca-config.json -profile=kubernetes \
-hostname=10.96.0.1,127.0.0.1,k8s.yunlearn.org,master01,master02,master03,kubernetes,kubernetes.default,kubernetes.default.svc,kubernetes.default.svc.cluster,kubernetes.default.svc.cluster.local,192.168.1.10,192.168.1.11,192.168.1.12 \
/root/kubernetes/certjson/etcd-server-csr.json | cfssljson -bare /data/etcd/certs/etcd
# 校验etcd server证书期限
openssl x509 -in /data/etcd/certs/etcd.pem -text -noout | grep Not 

5、签发etcd peer证书
cfssl gencert -ca=/data/etcd/certs/etcd-ca.pem -ca-key=/data/etcd/certs/etcd-ca-key.pem -config=/root/kubernetes/certjson/etcd-ca-config.json -profile=kubernetes \
-hostname=10.96.0.1,127.0.0.1,k8s.yunlearn.org,master01,master02,master03,kubernetes,kubernetes.default,kubernetes.default.svc,kubernetes.default.svc.cluster,kubernetes.default.svc.cluster.local,192.168.1.10,192.168.1.11,192.168.1.12 \
/root/kubernetes/certjson/etcd-peer-csr.json | cfssljson -bare /data/etcd/certs/peer
# 校验etcd peer证书期限
openssl x509 -in /data/etcd/certs/peer.pem -text -noout | grep Not

6、签发etcd client证书
cfssl gencert -ca=/data/etcd/certs/etcd-ca.pem -ca-key=/data/etcd/certs/etcd-ca-key.pem -config=/root/kubernetes/certjson/etcd-ca-config.json -profile=apiserver-etcd-client \
-hostname=10.96.0.1,127.0.0.1,k8s.yunlearn.org,master01,master02,master03,kubernetes,kubernetes.default,kubernetes.default.svc,kubernetes.default.svc.cluster,kubernetes.default.svc.cluster.local,192.168.1.10,192.168.1.11,192.168.1.12 \
/root/kubernetes/certjson/etcd-client-csr.json | cfssljson -bare /data/etcd/certs/apiserver-etcd-client
# 校验etcd client证书期限
openssl x509 -in /data/etcd/certs/apiserver-etcd-client.pem -text -noout | grep Not

3、配置etcd集群节点

1、开放端口 如果执行命令提示防火墙没运行，请启动防火墙再执行命令
firewall-cmd --get-active-zones
firewall-cmd --list-port
firewall-cmd --zone=public --permanent --add-port=2379/tcp --add-port=2380/tcp
firewall-cmd --reload
firewall-cmd --list-port

2、登录etcd1、etcd2、etcd3服务器拉取镜像
docker pull registry.aliyuncs.com/google_containers/etcd:3.5.1-0

3、登录etcd1、etcd2、etcd3服务器，在etcd2、etcd3创建目录，将 etcd1节点的证书拷贝到etcd2、etcd3节点
mkdir -p  /data/etcd/{certs,data}
cd /data/etcd/certs/
scp etcd-ca-key.pem etcd-ca.pem etcd-ca.csr etcd.csr etcd-key.pem etcd.pem peer.csr peer-key.pem peer.pem root@192.168.1.11:/data/etcd/certs/
scp etcd-ca-key.pem etcd-ca.pem etcd-ca.csr etcd.csr etcd-key.pem etcd.pem peer.csr peer-key.pem peer.pem root@192.168.1.12:/data/etcd/certs/

4、部署etcd集群

1、启动etcd1节点容器
server_1=192.168.1.10
server_2=192.168.1.11
server_3=192.168.1.12
etcd_1=etcd1 
etcd_2=etcd2 
etcd_3=etcd3 
client_port=2379  peer_port=2380 

docker run -d --net=host --restart=always --name=${etcd_1} \
-v /data/etcd/certs:/certs \
-v /data/etcd/data/:/var/lib/etcd \
registry.aliyuncs.com/google_containers/etcd:3.5.1-0 \
etcd -name=${etcd_1} \
--listen-peer-urls=https://${server_1}:${peer_port} \
--listen-client-urls=https://${server_1}:${client_port},https://127.0.0.1:${client_port} \
--advertise-client-urls=https://${server_1}:${client_port} \
--initial-advertise-peer-urls=https://${server_1}:${peer_port} \
--initial-cluster-token=learn-etcd-cluster \
--initial-cluster=${etcd_1}=https://${server_1}:${peer_port},${etcd_2}=https://${server_2}:${peer_port},${etcd_3}=https://${server_3}:${peer_port} \
--initial-cluster-state=new \
--trusted-ca-file=/certs/etcd-ca.pem \
--auto-tls=true \
--data-dir=/var/lib/etcd \
--cert-file=/certs/etcd.pem \
--key-file=/certs/etcd-key.pem \
--client-cert-auth=true \
--peer-trusted-ca-file=/certs/etcd-ca.pem \
--peer-auto-tls=true \
--peer-cert-file=/certs/peer.pem \
--peer-key-file=/certs/peer-key.pem \
--peer-client-cert-auth=true \
--election-timeout=10000 \
--heartbeat-interval=2000 \
--auto-compaction-mode=revision \
--auto-compaction-retention=24 \
--max-request-bytes=33554432 \
--quota-backend-bytes=8589934592 \
--snapshot-count=10000 

2、启动etcd2节点容器
server_1=192.168.1.10
server_2=192.168.1.11
server_3=192.168.1.12
etcd_1=etcd1 
etcd_2=etcd2 
etcd_3=etcd3 
client_port=2379  peer_port=2380 

docker run -d --net=host --restart=always --name=${etcd_2} \
-v /data/etcd/certs:/certs \
-v /data/etcd/data/:/var/lib/etcd \
registry.aliyuncs.com/google_containers/etcd:3.5.1-0 \
etcd -name=${etcd_2} \
--listen-peer-urls=https://${server_2}:${peer_port} \
--listen-client-urls=https://${server_2}:${client_port},https://127.0.0.1:${client_port} \
--advertise-client-urls=https://${server_2}:${client_port} \
--initial-advertise-peer-urls=https://${server_2}:${peer_port} \
--initial-cluster-token=learn-etcd-cluster \
--initial-cluster=${etcd_1}=https://${server_1}:${peer_port},${etcd_2}=https://${server_2}:${peer_port},${etcd_3}=https://${server_3}:${peer_port} \
--initial-cluster-state=new \
--trusted-ca-file=/certs/etcd-ca.pem \
--auto-tls=true \
--data-dir=/var/lib/etcd \
--cert-file=/certs/etcd.pem \
--key-file=/certs/etcd-key.pem \
--client-cert-auth=true \
--peer-trusted-ca-file=/certs/etcd-ca.pem \
--peer-auto-tls=true \
--peer-cert-file=/certs/peer.pem \
--peer-key-file=/certs/peer-key.pem \
--peer-client-cert-auth=true \
--election-timeout=10000 \
--heartbeat-interval=2000 \
--auto-compaction-mode=revision \
--auto-compaction-retention=24 \
--max-request-bytes=33554432 \
--quota-backend-bytes=8589934592 \
--snapshot-count=10000

3、启动etcd3节点容器
server_1=192.168.1.10
server_2=192.168.1.11
server_3=192.168.1.12
etcd_1=etcd1
etcd_2=etcd2
etcd_3=etcd3
client_port=2379 peer_port=2380

docker run -d --net=host --restart=always --name=${etcd_3} \
-v /data/etcd/certs:/certs \
-v /data/etcd/data/:/var/lib/etcd \
registry.aliyuncs.com/google_containers/etcd:3.5.1-0 \
etcd -name=${etcd_3} \
--listen-peer-urls=https://${server_3}:${peer_port} \
--listen-client-urls=https://${server_3}:${client_port},https://127.0.0.1:${client_port} \
--advertise-client-urls=https://${server_3}:${client_port} \
--initial-advertise-peer-urls=https://${server_3}:${peer_port} \
--initial-cluster-token=learn-etcd-cluster \
--initial-cluster=${etcd_1}=https://${server_1}:${peer_port},${etcd_2}=https://${server_2}:${peer_port},${etcd_3}=https://${server_3}:${peer_port} \
--initial-cluster-state=new \
--trusted-ca-file=/certs/etcd-ca.pem \
--auto-tls=true \
--data-dir=/var/lib/etcd \
--cert-file=/certs/etcd.pem \
--key-file=/certs/etcd-key.pem \
--client-cert-auth=true \
--peer-trusted-ca-file=/certs/etcd-ca.pem \
--peer-auto-tls=true \
--peer-cert-file=/certs/peer.pem \
--peer-key-file=/certs/peer-key.pem \
--peer-client-cert-auth=true \
--election-timeout=10000 \
--heartbeat-interval=2000 \
--auto-compaction-mode=revision \
--auto-compaction-retention=24 \
--max-request-bytes=33554432 \
--quota-backend-bytes=8589934592 \
--snapshot-count=10000

# 优化参数说明
--election-timeout=10000 \ #选主超时时间10秒
--heartbeat-interval=2000 \ #节点心跳时间2秒
--auto-compaction-mode=revision \ #版本压缩
--auto-compaction-retention=24 \ #启用压缩，保留24小时
--max-request-bytes=33554432 \ #单条记录32M
--quota-backend-bytes=8589934592 \ #存储配额8G

5、验证etcd集群

1、按顺序执行验证命令
# 确认三个节点etcd容器运行正常
docker ps 
docker logs -f --tail=200 <containerid>

2、进入容器，验证集群状态
docker exec -it etcd1 sh
# 设置etcdctl为v3版本
export ETCDCTL_API=3
# 3.4版及以上，需要设置证书才能执行维护命令
alias etcdctl='etcdctl --endpoints=https://127.0.0.1:2379 --cacert=/certs/etcd-ca.pem --cert=/certs/etcd.pem --key=/certs/etcd-key.pem'
# 查看集群
etcdctl member list
etcdctl endpoint health
etcdctl endpoint status 

3、读写数据
etcdctl put /learn dataTest
etcdctl get /learn 
etcdctl del /learn

6、更多k8s学习资料

​​1、kubernetes原理精讲【基础原理+实践篇】​​

​​2、kubernetes原理精讲【自签证书原理+实践篇】​​