etcd安装-CFANZ编程社区

1、单节点部署

1.1、安装

# centos7环境安装etcd
#
# 下载etcd
wget https://github.com/etcd-io/etcd/releases/download/v3.5.11/etcd-v3.5.11-linux-amd64.tar.gz
 
# 解压
tar -xf etcd-v3.5.11-linux-amd64.tar.gz
 
# 进入 etcd-v3.5.11-linux-amd64目录
cd etcd-v3.5.11-linux-amd64
 
# 将etcd、etcdctl、etcdutl移动到/usr/local/bin目录
mv etcd /usr/local/bin/etcd
mv etcdctl /usr/local/bin/etcdctl
mv etcdutl /usr/local/bin/etcdutl
 
# 查看etcd版本
etcd --version
etcdctl version
etcdutl version

1.2、运行

# 运行etcd
# 会在当前目录创建data-dir="default.etcd"，wal-dir="default.etcd/member/wal"
etcd

# 指定参数运行
etcd --listen-client-urls=http://$PRIVATE_IP:2379 \
   --advertise-client-urls=http://$PRIVATE_IP:2379


# 查看集群状态
etcdctl --endpoints=$ENDPOINTS endpoint status
etcdctl --endpoints=$ENDPOINTS endpoint health

1.3、配置选项

1.3.1、使用命令行参数启动etcd

# 使用命令行参数运行etcd
etcd --name 'command-line-flags' --data-dir '/root/command-line-flags'

1.3.2、使用环境变量启动etcd

创建/root/etcd.conf配置文件：

cat >  /root/etcd.conf <<"EOF"
#[Member]
ETCD_NAME='environment-variable'
ETCD_DATA_DIR='/root/environment-variable'
EOF

创建/etc/systemd/system/etcd.service服务配置文件：

cat > /etc/systemd/system/etcd.service <<"EOF"
[Unit]
Description=Etcd Server
After=network.target
After=network-online.target
Wants=network-online.target

[Service]
Type=notify
EnvironmentFile=/root/etcd.conf
ExecStart=/usr/local/bin/etcd
Restart=on-failure
RestartSec=5
LimitNOFILE=65536

[Install]
WantedBy=multi-user.target
EOF

启动Systemd服务：

# systemctl daemon-reload && systemctl enable --now etcd.service
systemctl daemon-reload && systemctl enable etcd && systemctl start etcd

systemctl status etcd

1.3.3、使用配置文件启动etcd

下载etcd.config.yml配置文件：

# 下载配置文件
wget https://github.com/etcd-io/etcd/blob/main/etcd.conf.yml.sample -O /root/etcd.config.yml

查看/root/etcd.config.yml文件：

# This is the configuration file for the etcd server.

# Human-readable name for this member.
name: 'default'

# Path to the data directory.
data-dir:

# Path to the dedicated wal directory.
wal-dir:

# Number of committed transactions to trigger a snapshot to disk.
snapshot-count: 10000

# Time (in milliseconds) of a heartbeat interval.
heartbeat-interval: 100

# Time (in milliseconds) for an election to timeout.
election-timeout: 1000

# Raise alarms when backend size exceeds the given quota. 0 means use the
# default quota.
quota-backend-bytes: 0

# List of comma separated URLs to listen on for peer traffic.
listen-peer-urls: http://localhost:2380

# List of comma separated URLs to listen on for client traffic.
listen-client-urls: http://localhost:2379

# Maximum number of snapshot files to retain (0 is unlimited).
max-snapshots: 5

# Maximum number of wal files to retain (0 is unlimited).
max-wals: 5

# Comma-separated white list of origins for CORS (cross-origin resource sharing).
cors:

# List of this member's peer URLs to advertise to the rest of the cluster.
# The URLs needed to be a comma-separated list.
initial-advertise-peer-urls: http://localhost:2380

# List of this member's client URLs to advertise to the public.
# The URLs needed to be a comma-separated list.
advertise-client-urls: http://localhost:2379

# Discovery URL used to bootstrap the cluster.
discovery:

# Valid values include 'exit', 'proxy'
discovery-fallback: 'proxy'

# HTTP proxy to use for traffic to discovery service.
discovery-proxy:

# DNS domain used to bootstrap initial cluster.
discovery-srv:

# Comma separated string of initial cluster configuration for bootstrapping.
# Example: initial-cluster: "infra0=http://10.0.1.10:2380,infra1=http://10.0.1.11:2380,infra2=http://10.0.1.12:2380"
initial-cluster:

# Initial cluster token for the etcd cluster during bootstrap.
initial-cluster-token: 'etcd-cluster'

# Initial cluster state ('new' or 'existing').
initial-cluster-state: 'new'

# Reject reconfiguration requests that would cause quorum loss.
strict-reconfig-check: false

# Enable runtime profiling data via HTTP server
enable-pprof: true

# Valid values include 'on', 'readonly', 'off'
proxy: 'off'

# Time (in milliseconds) an endpoint will be held in a failed state.
proxy-failure-wait: 5000

# Time (in milliseconds) of the endpoints refresh interval.
proxy-refresh-interval: 30000

# Time (in milliseconds) for a dial to timeout.
proxy-dial-timeout: 1000

# Time (in milliseconds) for a write to timeout.
proxy-write-timeout: 5000

# Time (in milliseconds) for a read to timeout.
proxy-read-timeout: 0

client-transport-security:
  # Path to the client server TLS cert file.
  cert-file:

  # Path to the client server TLS key file.
  key-file:

  # Enable client cert authentication.
  client-cert-auth: false

  # Path to the client server TLS trusted CA cert file.
  trusted-ca-file:

  # Client TLS using generated certificates
  auto-tls: false

peer-transport-security:
  # Path to the peer server TLS cert file.
  cert-file:

  # Path to the peer server TLS key file.
  key-file:

  # Enable peer client cert authentication.
  client-cert-auth: false

  # Path to the peer server TLS trusted CA cert file.
  trusted-ca-file:

  # Peer TLS using generated certificates.
  auto-tls: false

# The validity period of the self-signed certificate, the unit is year.
self-signed-cert-validity: 1

# Enable debug-level logging for etcd.
log-level: debug

logger: zap

# Specify 'stdout' or 'stderr' to skip journald logging even when running under systemd.
log-outputs: [stderr]

# Force to create a new one member cluster.
force-new-cluster: false

auto-compaction-mode: periodic
auto-compaction-retention: "1"

# Limit etcd to a specific set of tls cipher suites
cipher-suites: [
  TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,
  TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384
]

# Limit etcd to specific TLS protocol versions 
tls-min-version: 'TLS1.2'
tls-max-version: 'TLS1.3'

编辑/root/etcd.config.yml文件：

# 需要修改以下字段
# name
# data-dir
# wal-dir
# listen-peer-urls
# listen-client-urls
# initial-advertise-peer-urls
# advertise-client-urls
# initial-cluster
# initial-cluster-token
# client-transport-security、client-transport-security下级字段
# peer-transport-security、peer-transport-security下级字段


name: 'configuration-file'
data-dir: '/root/configuration-file'
wal-dir:
snapshot-count: 10000
heartbeat-interval: 100
election-timeout: 1000
quota-backend-bytes: 0
listen-peer-urls: http://localhost:2380
listen-client-urls: http://localhost:2379
max-snapshots: 5
max-wals: 5
cors:
initial-advertise-peer-urls: http://localhost:2380
advertise-client-urls: http://localhost:2379
discovery:
discovery-fallback: 'proxy'
discovery-proxy:
discovery-srv:
initial-cluster:
initial-cluster-token: 'etcd-cluster'
initial-cluster-state: 'new'
strict-reconfig-check: false
enable-pprof: true
proxy: 'off'
proxy-failure-wait: 5000
proxy-refresh-interval: 30000
proxy-dial-timeout: 1000
proxy-write-timeout: 5000
proxy-read-timeout: 0
client-transport-security:
  cert-file:
  key-file:
  client-cert-auth: false
  trusted-ca-file:
  auto-tls: false
peer-transport-security:
  cert-file:
  key-file:
  client-cert-auth: false
  trusted-ca-file:
  auto-tls: false
self-signed-cert-validity: 1
log-level: debug
logger: zap
log-outputs: [stderr]
force-new-cluster: false
auto-compaction-mode: periodic
auto-compaction-retention: "1"
cipher-suites: [
  TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,
  TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384
]
tls-min-version: 'TLS1.2'
tls-max-version: 'TLS1.3'

执行命令：

# 使用配置文件运行etcd
etcd --config-file /root/etcd.config.yml

2、集群部署

2.1、使用http通信

172.17.0.2节点：

etcd --data-dir=data.etcd --name etcd-node-1 \
	--initial-advertise-peer-urls http://172.17.0.2:2380 \
	--listen-peer-urls http://172.17.0.2:2380 \
	--advertise-client-urls http://172.17.0.2:2379 \
	--listen-client-urls http://172.17.0.2:2379 \
	--initial-cluster etcd-node-1=http://172.17.0.2:2380,etcd-node-2=http://172.17.0.3:2380,etcd-node-3=http://172.17.0.4:2380 \
	--initial-cluster-state new \
	--initial-cluster-token etcd-token \
    --peer-key-file peer.key \
    --peer-cert-file peer.cert

172.17.0.3节点：

etcd --data-dir=data.etcd --name etcd-node-2 \
	--initial-advertise-peer-urls http://172.17.0.3:2380 \
	--listen-peer-urls http://172.17.0.3:2380 \
	--advertise-client-urls http://172.17.0.3:2379 \
	--listen-client-urls http://172.17.0.3:2379 \
	--initial-cluster etcd-node-1=http://172.17.0.2:2380,etcd-node-2=http://172.17.0.3:2380,etcd-node-3=http://172.17.0.4:2380 \
	--initial-cluster-state new \
	--initial-cluster-token etcd-token

172.17.0.4节点：

etcd --data-dir=data.etcd --name etcd-node-3 \
	--initial-advertise-peer-urls http://172.17.0.4:2380 \
	--listen-peer-urls http://172.17.0.4:2380 \
	--advertise-client-urls http://172.17.0.4:2379 \
	--listen-client-urls http://172.17.0.4:2379 \
	--initial-cluster etcd-node-1=http://172.17.0.2:2380,etcd-node-2=http://172.17.0.3:2380,etcd-node-3=http://172.17.0.4:2380 \
	--initial-cluster-state new \
	--initial-cluster-token etcd-token

172.17.0.2节点，查看集群：

# 1、查看集群
etcdctl --endpoints=172.17.0.2:2379,172.17.0.3:2379,172.17.0.4:2379 member list

# 显示结果
17396d3aa5468ea, started, etcd-node-1, http://172.17.0.2:2380, http://172.17.0.2:2379, false
3a9e24ab09adc359, started, etcd-node-2, http://172.17.0.3:2380, http://172.17.0.3:2379, false
c79b72d1b9e2d891, started, etcd-node-3, http://172.17.0.4:2380, http://172.17.0.4:2379, false

# ===============================================================================
# 2、移除节点
etcdctl --endpoints=172.17.0.2:2379,172.17.0.3:2379,172.17.0.4:2379 member remove c79b72d1b9e2d891

# 显示结果
Member c79b72d1b9e2d891 removed from cluster 8cc616c226560de7

# 3、查看集群
etcdctl --endpoints=172.17.0.2:2379,172.17.0.3:2379,172.17.0.4:2379 member list

# 显示结果
17396d3aa5468ea, started, etcd-node-1, http://172.17.0.2:2380, http://172.17.0.2:2379, false
3a9e24ab09adc359, started, etcd-node-2, http://172.17.0.3:2380, http://172.17.0.3:2379, false


# ===============================================================================
# 4、添加节点
etcdctl --endpoints=172.17.0.2:2379,172.17.0.3:2379 member add etcd-node-3 --peer-urls=http://172.17.0.4:2380

# 显示结果
Member 99d9b4191b7df3f2 added to cluster 8cc616c226560de7

ETCD_NAME="etcd-node-3"
ETCD_INITIAL_CLUSTER="etcd-node-1=http://172.17.0.2:2380,etcd-node-2=http://172.17.0.3:2380,etcd-node-3=http://172.17.0.4:2380"
ETCD_INITIAL_ADVERTISE_PEER_URLS="http://172.17.0.4:2380"
ETCD_INITIAL_CLUSTER_STATE="existing"

# 5、查看集群
etcdctl --endpoints=172.17.0.2:2379,172.17.0.3:2379,172.17.0.4:2379 member list

# 显示结果
17396d3aa5468ea, started, etcd-node-1, http://172.17.0.2:2380, http://172.17.0.2:2379, false
3a9e24ab09adc359, started, etcd-node-2, http://172.17.0.3:2380, http://172.17.0.3:2379, false
91fd2ad3e8b7b64f, unstarted, , http://172.17.0.4:2380, , false


# 6、172.17.0.4节点执行
rm -rf ./etcd-node-3.etcd

# 7、172.17.0.4节点执行
etcd --data-dir=data.etcd --name etcd-node-3 \
	--initial-advertise-peer-urls http://172.17.0.4:2380 \
	--listen-peer-urls http://172.17.0.4:2380 \
	--advertise-client-urls http://172.17.0.4:2379 \
	--listen-client-urls http://172.17.0.4:2379 \
	--initial-cluster etcd-node-1=http://172.17.0.2:2380,etcd-node-2=http://172.17.0.3:2380,etcd-node-3=http://172.17.0.4:2380 \
	--initial-cluster-state existing \
	--initial-cluster-token etcd-token

# 8、查看集群
etcdctl --endpoints=172.17.0.2:2379,172.17.0.3:2379,172.17.0.4:2379 member list

# 显示结果
17396d3aa5468ea, started, etcd-node-1, http://172.17.0.2:2380, http://172.17.0.2:2379, false
2e6bc6b95d92f714, started, etcd-node-3, http://172.17.0.4:2380, http://172.17.0.4:2379, false
3a9e24ab09adc359, started, etcd-node-2, http://172.17.0.3:2380, http://172.17.0.3:2379, false

2.2、使用https通信

安装cfssl

# 1、下载cfssl、cfssljson、cfssl-certinfo
# cfssl：用于签发证书
# cfssljson：将cfssl签发生成的证书(json格式)变成文件承载式文件
# cfssl-certinfo：验证查看证书信息
wget https://pkg.cfssl.org/R1.2/cfssl_linux-amd64 -O /usr/local/bin/cfssl
wget https://pkg.cfssl.org/R1.2/cfssljson_linux-amd64 -O /usr/local/bin/cfssljson
wget https://pkg.cfssl.org/R1.2/cfssl-certinfo_linux-amd64 -O /usr/local/bin/cfssl-certinfo
 
# 2、给cfssl、cfssljson、cfssl-certinfo添加可执行权限
chmod +x /usr/local/bin/cfssl*

生成证书

ca-config.json文件：

{
  "signing": {
    "default": {
        "usages": [
          "signing",
          "key encipherment",
          "server auth",
          "client auth"
        ],
        "expiry": "876000h"
    }
  }
}

ca-csr.json文件：

{
  "CN": "Autogenerated CA",
  "key": {
    "algo": "rsa",
    "size": 2048
  },
  "names": [
    {
      "O": "Honest Achmed's Used Certificates",
      "OU": "Hastily-Generated Values Divison",
      "L": "San Francisco",
      "ST": "California",
      "C": "US"
    }
  ]
}

etcd-csr.json文件：

{
  "CN": "etcd",
  "hosts": [
    "localhost",
    "127.0.0.1",
    "172.17.0.2",
    "172.17.0.3",
    "172.17.0.4"
  ],
  "key": {
    "algo": "rsa",
    "size": 2048
  },
  "names": [
    {
      "O": "autogenerated",
      "OU": "etcd cluster",
      "L": "the internet"
    }
  ]
}

执行命令：

# 创建目录
mkdir -p etcd/ssl

# 切换目录
cd etcd/ssl

# 创建文件，复制上述ca-config.json、ca-csr.json、etcd-csr.json内容到对应文件
touch {ca-config.json,ca-csr.json,etcd-csr.json}

# 生成CA根证书及其私钥
cfssl gencert -initca ca-csr.json | cfssljson -bare ca

# 根据CA根证书及其私钥签名生成目标证书和私钥
cfssl gencert -ca ca.pem -ca-key ca-key.pem -config ca-config.json etcd-csr.json | cfssljson -bare etcd

# 复制etcd目录到其他节点
scp -r /root/etcd 172.17.0.3:/root/etcd
scp -r /root/etcd 172.17.0.4:/root/etcd

172.17.0.2节点：

etcd --data-dir=data.etcd --name etcd-node-1 \
    --initial-advertise-peer-urls https://172.17.0.2:2380 \
    --listen-peer-urls https://172.17.0.2:2380 \
    --advertise-client-urls https://172.17.0.2:2379 \
    --listen-client-urls https://172.17.0.2:2379 \
    --initial-cluster 'etcd-node-1=https://172.17.0.2:2380,etcd-node-2=https://172.17.0.3:2380,etcd-node-3=https://172.17.0.4:2380' \
    --initial-cluster-state new \
    --initial-cluster-token etcd-token \
    --cert-file=/root/etcd/ssl/etcd.pem \
    --key-file=/root/etcd/ssl/etcd-key.pem \
    --peer-cert-file=/root/etcd/ssl/etcd.pem \
    --peer-key-file=/root/etcd/ssl/etcd-key.pem \
    --peer-client-cert-auth \
    --peer-trusted-ca-file=/root/etcd/ssl/ca.pem

172.17.0.3节点：

etcd --data-dir=data.etcd --name etcd-node-2 \
    --initial-advertise-peer-urls https://172.17.0.3:2380 \
    --listen-peer-urls https://172.17.0.3:2380 \
    --advertise-client-urls https://172.17.0.3:2379 \
    --listen-client-urls https://172.17.0.3:2379 \
    --initial-cluster 'etcd-node-1=https://172.17.0.2:2380,etcd-node-2=https://172.17.0.3:2380,etcd-node-3=https://172.17.0.4:2380' \
    --initial-cluster-state new \
    --initial-cluster-token etcd-token \
    --cert-file=/root/etcd/ssl/etcd.pem \
    --key-file=/root/etcd/ssl/etcd-key.pem \
    --peer-cert-file=/root/etcd/ssl/etcd.pem \
    --peer-key-file=/root/etcd/ssl/etcd-key.pem \
    --peer-client-cert-auth \
    --peer-trusted-ca-file=/root/etcd/ssl/ca.pem

172.17.0.4节点：

etcd --data-dir=data.etcd --name etcd-node-3 \
    --initial-advertise-peer-urls https://172.17.0.4:2380 \
    --listen-peer-urls https://172.17.0.4:2380 \
    --advertise-client-urls https://172.17.0.4:2379 \
    --listen-client-urls https://172.17.0.4:2379 \
    --initial-cluster 'etcd-node-1=https://172.17.0.2:2380,etcd-node-2=https://172.17.0.3:2380,etcd-node-3=https://172.17.0.4:2380' \
    --initial-cluster-state new \
    --initial-cluster-token etcd-token \
    --cert-file=/root/etcd/ssl/etcd.pem \
    --key-file=/root/etcd/ssl/etcd-key.pem \
    --peer-cert-file=/root/etcd/ssl/etcd.pem \
    --peer-key-file=/root/etcd/ssl/etcd-key.pem \
    --peer-client-cert-auth \
    --peer-trusted-ca-file=/root/etcd/ssl/ca.pem

172.17.0.2节点，查看集群：

# 查看集群
etcdctl --endpoints=172.17.0.2:2379,172.17.0.3:2379,172.17.0.4:2379 \
  --cert=/root/etcd/ssl/etcd.pem \
  --key=/root/etcd/ssl/etcd-key.pem \
  --cacert=/root/etcd/ssl/ca.pem \
  member list

# 显示结果
c6b958fbe52963, started, etcd-node-1, https://172.17.0.2:2380, https://172.17.0.2:2379, false
5f334165954101b, started, etcd-node-3, https://172.17.0.4:2380, https://172.17.0.4:2379, false
964941aff35ec5da, started, etcd-node-2, https://172.17.0.3:2380, https://172.17.0.3:2379, false


# 不带证书访问集群
etcdctl --endpoints=172.17.0.2:2379,172.17.0.3:2379,172.17.0.4:2379   member list

# 显示结果
{"level":"warn","ts":"2024-01-20T21:58:12.352461Z","logger":"etcd-client","caller":"v3@v3.5.11/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000374380/172.17.0.2:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"error reading server preface: EOF\""}
Error: context deadline exceeded

详见：

操作 Kubernetes 中的 etcd 集群 | Kubernetes

Clustering Guide | etcd

How to Set Up a Demo etcd Cluster | etcd

How to Add and Remove Members | etcd

3、docker部署

3.1、使用http通信

# 当前主机IP：10.0.8.13
# 创建目录
mkdir /root/etcd-data


# 当前主机IP：10.0.8.13
# 创建容器
docker run \
  -p 2379:2379 \
  -p 2380:2380 \
  --volume=/root/etcd-data:/root/etcd-data \
  --name etcd quay.io/coreos/etcd:latest \
  /usr/local/bin/etcd \
  --data-dir=/root/etcd-data --name node1 \
  --initial-advertise-peer-urls http://10.0.8.13:2380 \
  --listen-peer-urls http://0.0.0.0:2380 \
  --advertise-client-urls http://10.0.8.13:2379 \
  --listen-client-urls http://0.0.0.0:2379 \
  --initial-cluster node1=http://10.0.8.13:2380


# 当前主机IP：10.0.8.13
# 查看集群状态
etcdctl --endpoints=10.0.8.13:2379 member list

# 显示结果
942908011bbade83, started, node1, http://10.0.8.13:2380, http://10.0.8.13:2379, false

3.2、使用https通信

# 当前主机IP：10.0.8.13
# 创建目录
mkdir -p /root/etcd/{data,ssl}

# 生成证书
# 参考：“2.2、使用https通信” 的 “生成证书”
# 修改“etcd-csr.json文件” 的 “hosts” 字段，hosts值应为 “主机IP”

# 当前主机IP：10.0.8.13
# 创建容器
docker run \
  -p 2379:2379 \
  -p 2380:2380 \
  --volume=/root/etcd:/root/etcd \
  --name etcd quay.io/coreos/etcd:latest \
  /usr/local/bin/etcd \
  --data-dir=/root/etcd/data --name node1 \
  --initial-advertise-peer-urls https://10.0.8.13:2380 \
  --listen-peer-urls https://0.0.0.0:2380 \
  --advertise-client-urls https://10.0.8.13:2379 \
  --listen-client-urls https://0.0.0.0:2379 \
  --cert-file=/root/etcd/ssl/etcd.pem \
  --key-file=/root/etcd/ssl/etcd-key.pem \
  --peer-cert-file=/root/etcd/ssl/etcd.pem \
  --peer-key-file=/root/etcd/ssl/etcd-key.pem \
  --peer-client-cert-auth \
  --peer-trusted-ca-file=/root/etcd/ssl/ca.pem


# 当前主机IP：10.0.8.13
# 查看集群
etcdctl --endpoints=10.0.8.13:2379 \
  --cert=/root/etcd/ssl/etcd.pem \
  --key=/root/etcd/ssl/etcd-key.pem \
  --cacert=/root/etcd/ssl/ca.pem \
  member list

# 显示结果
b0b9626eea1088ab, started, node1, https://10.0.8.13:2380, https://10.0.8.13:2379, false


# 当前主机IP：10.0.8.13
# 不带证书访问集群
etcdctl --endpoints=10.0.8.13:2379 member list

# 显示结果
{"level":"warn","ts":"2024-01-21T15:42:38.997455+0800","logger":"etcd-client","caller":"v3@v3.5.11/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc00007c000/10.0.8.13:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"error reading server preface: read tcp 10.0.8.13:56464->10.0.8.13:2379: read: connection reset by peer\""}
Error: context deadline exceeded