第二十一周学习作业-CFANZ编程社区

1、简述redis集群的实现原理

在哨兵sentinel机制中，可以解决redis高可用问题，即当master故障后可以自动将slave提升为master，从而可以保证redis服务的正常使用，但是无法解决redis单机写入的瓶颈问题，即单机redis写入性能受限于单机的内存大小、并发数量、网卡速率等因素。

为了解决单机性能的瓶颈，提高Redis 性能，可以使用分布式集群的解决方案

早期Redis 分布式集群部署方案：

客户端分区：由客户端程序决定key写分配和写入的redis node，但是需要客户端自己实现写入分配、高可用管理和故障转移等
代理方案：基于三方软件实现redis proxy，客户端先连接之代理层，由代理层实现key的写入分配，对客户端来说是有比较简单，但是对于集群管节点增减相对比较麻烦，而且代理本身也是单点和性能瓶颈。

redis 3.0版本之后推出了无中心架构的redis cluster机制，在无中心的redis集群当中，其每个节点保存当前节点数据和整个集群状态,每个节点都和其他所有节点连接

Redis Cluster特点如下：

所有Redis节点使用(PING机制)互联
集群中某个节点的是否失效，是由整个集群中超过半数的节点监测都失效，才能算真正的失效
客户端不需要proxy即可直接连接redis，应用程序中需要配置有全部的redis服务器IP
redis cluster把所有的redis node 平均映射到 0-16383个槽位(slot)上，读写需要到指定的redis node上进行操作，因此有多少个redis node相当于redis 并发扩展了多少倍，每个redis node 承担16384/N个槽位
Redis cluster预先分配16384个(slot)槽位，当需要在redis集群中写入一个key -value的时候，会使用CRC16(key) mod 16384之后的值，决定将key写入值哪一个槽位从而决定写入哪一个Redis节点上，从而有效解决单机瓶颈。

2、基于redis5的redis cluster部署

环境说明(本实验用的是版本6.2.4)

第二十一周学习作业_redis

2.1 部署多实例redis

## 每个节点都执行
##一键编译安装6379端口redis实例
#!/bin/bash

VERSION=redis-6.2.4
PASSWORD=123456
INSTALL_DIR=/apps/redis

install() {
        yum install -y gcc jemalloc-devel wget make &>/dev/null
        if [ $? -eq 0 ] ;then
            echo "依赖包给成功部署！！"
        else
             echo "依赖包部署失败！！"
             exit 0
        fi
        echo "redis软件下载中..."
        wget http://download.redis.io/releases/${VERSION}.tar.gz &>/dev/null
        if [ $? -eq 0 ];then
                echo "redis软件下载完成！"
        else
                echo "请检查网络！！"
                exit 0
        fi

        tar xf ${VERSION}.tar.gz
        cd ${VERSION}
        echo "编译进行中..."
        make PREFIX=${INSTALL_DIR} install  &> /tmp/redis_make.log
        if [ $? -eq 0 ];then
             echo "编译成功！！"
                 rm -rf /tmp/redis_make.log
        else
             echo "编译失败！,请查看/tmp/redis_make.log！！"
             exit 0
        fi

        ln -s ${INSTALL_DIR}/bin/redis-* /usr/bin/
        mkdir -p ${INSTALL_DIR}/{etc,log,data,run}
        cp redis.conf ${INSTALL_DIR}/etc/redis_6379.conf

        sed -i -e 's/bind 127.0.0.1/bind 0.0.0.0/' -e "/# requirepass/a requirepass $PASSWORD" -e "/^dir .*/c dir ${INSTALL_DIR}/data/" -e '/^dbfilename dump.rdb/c dbfilename dump_6379.rdb' -e "/logfile .*/c logfile ${INSTALL_DIR}/log/redis_6379.log" -e "/^pidfile .*/c pidfile ${INSTALL_DIR}/run/redis_6379.pid" ${INSTALL_DIR}/etc/redis_6379.conf

        id redis &>/dev/null
        if [ $? -eq 0 ]; then
            echo "Redis 用户已存在!"
        else
            useradd -r -s /sbin/nologin redis
            echo "Redis 用户创建成功!"
        fi
        chown -R redis:redis ${INSTALL_DIR}

        cat >> /etc/sysctl.conf <<EOF
net.core.somaxconn = 1024
vm.overcommit_memory = 1
EOF
        sysctl -p
        echo 'echo never > /sys/kernel/mm/transparent_hugepage/enabled' >> /etc/rc.d/rc.local
        chmod +x /etc/rc.d/rc.local
        /etc/rc.d/rc.local
        cat > /usr/lib/systemd/system/redis_6379.service <<EOF
[Unit]
Description=Redis persistent key-value database
After=network.target

[Service]
ExecStart=${INSTALL_DIR}/bin/redis-server ${INSTALL_DIR}/etc/redis_6379.conf --supervised systemd
ExecStop=/bin/kill -s QUIT \$MAINPID
#Type=notify
User=redis
Group=redis
RuntimeDirectory=redis
RuntimeDirectoryMode=0755

[Install]
WantedBy=multi-user.target

EOF
        systemctl daemon-reload
        systemctl start redis_6379
        sleep 3
        redis-cli -p 6379 -a $PASSWORD INFO Server 2> /dev/null
        if [ $? -eq 0 ] ;then
          echo "redis 服务安装并启动完成！！"
        else
          echo "redis 服务启动失败！！"
        fi

}
install
[root@redis-01 ~]# sh install_redis.sh
依赖包给成功部署！！
redis软件下载中...
redis软件下载完成！
编译进行中...
编译成功！！
Redis 用户创建成功!
net.core.somaxconn = 1024
vm.overcommit_memory = 1
# Server
redis_version:6.2.4
redis_git_sha1:00000000
redis_git_dirty:0
redis_build_id:fc1523280a0a76e6
redis_mode:standalone
os:Linux 3.10.0-1160.49.1.el7.x86_64 x86_64
arch_bits:64
multiplexing_api:epoll
atomicvar_api:atomic-builtin
gcc_version:4.8.5
process_id:6031
process_supervised:no
run_id:0563ee35bb09715c03413b7eada97cba16d3d5e1
tcp_port:6379
server_time_usec:1653127322441191
uptime_in_seconds:3
uptime_in_days:0
hz:10
configured_hz:10
lru_clock:8960154
executable:/apps/redis/bin/redis-server
config_file:/apps/redis/etc/redis_6379.conf
io_threads_active:0
redis 服务安装并启动完成！！

#启动配置6380端口redis实例
[root@redis-01 ~]# cp /apps/redis/etc/redis_6379.conf /apps/redis/etc/redis_6380.conf
[root@redis-01 ~]# sed -i -e 's/port 6379/port 6380/' -e "/^dir .*/c dir /apps/redis/data/" -e '/^dbfilename dump_6379.rdb/c dbfilename dump_6380.rdb' -e "/logfile .*/c logfile /apps/redis/log/redis_6380.log" -e "/^pidfile .*/c pidfile /apps/redis/run/redis_6380.pid" /apps/redis/etc/redis_6380.conf
[root@redis-01 ~]# cp /lib/systemd/system/redis_6379.service /lib/systemd/system/redis_6380.service
[root@redis-01 ~]# sed -i 's/6379/6380/' /lib/systemd/system/redis_6380.service
[root@redis-01 ~]# systemctl daemon-reload
[root@redis-01 ~]# systemctl start redis_6380.service
[root@redis-01 ~]# ss -nltp
State      Recv-Q Send-Q                    Local Address:Port                                   Peer Address:Port
LISTEN     0      511                                   *:6379                                              *:*                   users:(("redis-server",pid=6065,fd=6))
LISTEN     0      511                                   *:6380                                              *:*                   users:(("redis-server",pid=12729,fd=6))
LISTEN     0      128                                   *:22                                                *:*                   users:(("sshd",pid=1104,fd=3))
LISTEN     0      100                           127.0.0.1:25                                                *:*                   users:(("master",pid=1300,fd=13))
LISTEN     0      511                               [::1]:6379                                           [::]:*                   users:(("redis-server",pid=6065,fd=7))
LISTEN     0      511                               [::1]:6380                                           [::]:*                   users:(("redis-server",pid=12729,fd=7))
LISTEN     0      128                                [::]:22                                             [::]:*                   users:(("sshd",pid=1104,fd=4))
LISTEN     0      100                               [::1]:25                                             [::]:*                   users:(("master",pid=1300,fd=14))
[root@redis-01 ~]# tree /apps/redis/
/apps/redis/
├── bin
│   ├── redis-benchmark
│   ├── redis-check-aof -> redis-server
│   ├── redis-check-rdb -> redis-server
│   ├── redis-cli
│   ├── redis-sentinel -> redis-server
│   └── redis-server
├── data
│   ├── dump_6379.rdb
│   └── dump_6380.rdb
├── etc
│   ├── redis_6379.conf
│   └── redis_6380.conf
├── log
│   ├── redis_6379.log
│   └── redis_6380.log
└── run
    ├── redis_6379.pid
    └── redis_6380.pid

5 directories, 14 files

2.2 启用redis cluster 配置

##每个节点都进行修改配置文件，一键修改
sed -i.bak -e '/^# masterauth .*/a  masterauth 123456' -e '/^# cluster-enabled yes/a cluster-enabled yes' -e '/^# cluster-config-file nodes-6379.conf/a cluster-config-file nodes-6379.conf' -e '/^# cluster-require-full-coverage yes/a cluster-require-full-coverage no' /apps/redis/etc/redis_6379.conf
sed -i.bak -e '/^# masterauth .*/a  masterauth 123456' -e '/^# cluster-enabled yes/a cluster-enabled yes' -e '/^# cluster-config-file nodes-6379.conf/a cluster-config-file nodes-6380.conf' -e '/^# cluster-require-full-coverage yes/a cluster-require-full-coverage no' /apps/redis/etc/redis_6380.conf
##重启redis服务
[root@redis_01 ~]# systemctl restart redis_6379.service redis_6380.service
[root@redis_01 ~]# ss -nltp
State      Recv-Q Send-Q                    Local Address:Port                                   Peer Address:Port
LISTEN     0      511                                   *:16379                                             *:*                   users:(("redis-server",pid=14764,fd=9))
LISTEN     0      511                                   *:16380                                             *:*                   users:(("redis-server",pid=14765,fd=9))
LISTEN     0      511                                   *:6379                                              *:*                   users:(("redis-server",pid=14764,fd=6))
LISTEN     0      511                                   *:6380                                              *:*                   users:(("redis-server",pid=14765,fd=6))
LISTEN     0      128                                   *:22                                                *:*                   users:(("sshd",pid=1107,fd=3))
LISTEN     0      100                           127.0.0.1:25                                                *:*                   users:(("master",pid=1309,fd=13))
LISTEN     0      511                               [::1]:16379                                          [::]:*                   users:(("redis-server",pid=14764,fd=10))
LISTEN     0      511                               [::1]:16380                                          [::]:*                   users:(("redis-server",pid=14765,fd=10))
LISTEN     0      511                               [::1]:6379                                           [::]:*                   users:(("redis-server",pid=14764,fd=7))
LISTEN     0      511                               [::1]:6380                                           [::]:*                   users:(("redis-server",pid=14765,fd=7))
LISTEN     0      128                                [::]:22                                             [::]:*                   users:(("sshd",pid=1107,fd=4))
LISTEN     0      100                               [::1]:25                                             [::]:*                   users:(("master",pid=1309,fd=14))

[root@redis_01 ~]# ps -ef | grep redis
redis     14764      1  0 18:20 ?        00:00:00 /apps/redis/bin/redis-server 0.0.0.0:6379 [cluster]
redis     14765      1  0 18:20 ?        00:00:00 /apps/redis/bin/redis-server 0.0.0.0:6380 [cluster]
root      14776   6471  0 18:21 pts/0    00:00:00 grep --color=auto redis
[root@redis_01 ~]# tree /apps/redis/
/apps/redis/
├── bin
│   ├── redis-benchmark
│   ├── redis-check-aof -> redis-server
│   ├── redis-check-rdb -> redis-server
│   ├── redis-cli
│   ├── redis-sentinel -> redis-server
│   └── redis-server
├── data
│   ├── dump_6379.rdb
│   ├── dump_6380.rdb
│   ├── nodes-6379.conf
│   └── nodes-6380.conf
├── etc
│   ├── redis_6379.conf
│   ├── redis_6379.conf.bak
│   ├── redis_6380.conf
│   └── redis_6380.conf.bak
├── log
│   ├── redis_6379.log
│   └── redis_6380.log
└── run
    ├── redis_6379.pid
    └── redis_6380.pid

5 directories, 18 files

2.2.1 手动配置redis cluster功能

##执行meet 操作实现相互通信
[root@redis-01 ~]# redis-cli -h 10.10.10.21 -a 123456 --no-auth-warning cluster meet 10.10.10.22 6379
OK
[root@redis-01 ~]# redis-cli -h 10.10.10.21 -a 123456 --no-auth-warning cluster meet 10.10.10.23 6379
OK
[root@redis-01 ~]# redis-cli -h 10.10.10.21 -a 123456 --no-auth-warning cluster meet 10.10.10.21 6380
OK
[root@redis-01 ~]# redis-cli -h 10.10.10.21 -a 123456 --no-auth-warning cluster meet 10.10.10.22 6380
OK
[root@redis-01 ~]# redis-cli -h 10.10.10.21 -a 123456 --no-auth-warning cluster meet 10.10.10.23 6380
OK

[root@redis-01 ~]# redis-cli -h 10.10.10.21 -a 123456 --no-auth-warning cluster nodes
b1426dd91455998e04470338abe01be5292139e0 10.10.10.23:6379@16379 master - 0 1653139742432 4 connected
d658b014aa1b1741f795a7fd52c3d76fdfe4c01f 10.10.10.23:6380@16380 master - 0 1653139741387 0 connected
37a9ea7fe27e44345a7e1a56a3417d41ec783a3b 10.10.10.22:6379@16379 master - 0 1653139741000 3 connected
70a02a82092d8fd54c6e0b64090a6493ff83d4e3 10.10.10.21:6380@16380 master - 0 1653139740000 2 connected
bcaa3e9f692ebd8775e67cf411f51c47c3842e31 10.10.10.22:6380@16380 master - 0 1653139740340 5 connected
8b2d94e3bdb451883d4e32be15900262ebad1e70 10.10.10.21:6379@16379 myself,master - 0 1653139741000 1 connected

##指定6380节点为从节点，6379为主节点的主从关系
[root@redis-01 ~]# redis-cli -h 10.10.10.21 -p 6380 -a 123456 --no-auth-warning cluster replicate b1426dd91455998e04470338abe01be5                          292139e0
OK
[root@redis-01 ~]# redis-cli -h 10.10.10.22 -p 6380 -a 123456 --no-auth-warning cluster replicate 8b2d94e3bdb451883d4e32be15900262                          ebad1e70
OK
[root@redis-01 ~]# redis-cli -h 10.10.10.23 -p 6380 -a 123456 --no-auth-warning cluster replicate 37a9ea7fe27e44345a7e1a56a3417d41                          ec783a3b
OK
##查看主从关系
[root@redis-01 ~]# redis-cli -h 10.10.10.21 -p 6379 -a 123456 -c INFO replication
Warning: Using a password with '-a' or '-u' option on the command line interface may not be safe.
# Replication
role:master
connected_slaves:1
slave0:ip=10.10.10.22,port=6380,state=online,offset=1596,lag=1
master_failover_state:no-failover
master_replid:7df980580ee25815dbe544e599d6a5961e93152b
master_replid2:0000000000000000000000000000000000000000
master_repl_offset:1596
second_repl_offset:-1
repl_backlog_active:1
repl_backlog_size:1048576
repl_backlog_first_byte_offset:1
repl_backlog_histlen:1596
[root@redis-01 ~]# redis-cli -h 10.10.10.22 -p 6379 -a 123456 -c INFO replication
Warning: Using a password with '-a' or '-u' option on the command line interface may not be safe.
# Replication
role:master
connected_slaves:1
slave0:ip=10.10.10.23,port=6380,state=online,offset=1638,lag=1
master_failover_state:no-failover
master_replid:ed845e6f5882db7edaab34ef897769fa1d7cbb15
master_replid2:0000000000000000000000000000000000000000
master_repl_offset:1638
second_repl_offset:-1
repl_backlog_active:1
repl_backlog_size:1048576
repl_backlog_first_byte_offset:1
repl_backlog_histlen:1638
[root@redis-01 ~]# redis-cli -h 10.10.10.23 -p 6379 -a 123456 -c INFO replication
Warning: Using a password with '-a' or '-u' option on the command line interface may not be safe.
# Replication
role:master
connected_slaves:1
slave0:ip=10.10.10.21,port=6380,state=online,offset=1750,lag=0
master_failover_state:no-failover
master_replid:d513b2710ddc8c43b7acf356128ab2d807bd2b9b
master_replid2:0000000000000000000000000000000000000000
master_repl_offset:1750
second_repl_offset:-1
repl_backlog_active:1
repl_backlog_size:1048576
repl_backlog_first_byte_offset:1
repl_backlog_histlen:1750
##集群信息，未分配slots
[root@redis-01 ~]# redis-cli -h 10.10.10.21 -a 123456 --no-auth-warning cluster info
cluster_state:fail
cluster_slots_assigned:0   ##无槽位分配置
cluster_slots_ok:0
cluster_slots_pfail:0
cluster_slots_fail:0
cluster_known_nodes:6
cluster_size:0          ##无集群成员
cluster_current_epoch:5
cluster_my_epoch:3
cluster_stats_messages_ping_sent:161
cluster_stats_messages_pong_sent:168
cluster_stats_messages_meet_sent:5
cluster_stats_messages_sent:334
cluster_stats_messages_ping_received:168
cluster_stats_messages_pong_received:166
cluster_stats_messages_received:334

#创建添加slots的脚本
[root@redis-01 ~]# vim addslot.sh
#!/bin/bash

host=$1
port=$2
start=$3
end=$4
pass=123456

for slot in `seq ${start} ${end}` ; do
        echo slot:$slot
        redis-cli -h ${host} -p $port -a ${pass} --no-auth-warning cluster addslots ${slot}
done
##为三个master分配槽位,共16364/3=5,461.333333333333,平均每个master分配5,461个槽位
[root@redis-01 ~]# sh addslot.sh 10.10.10.21 6379 0 5461
[root@redis-01 ~]# sh addslot.sh 10.10.10.22 6379 5462 10922
[root@redis-01 ~]# sh addslot.sh 10.10.10.23 6379 10923 16383
##集群状态正常
[root@redis-01 ~]# redis-cli -h 10.10.10.21 -a 123456 --no-auth-warning cluster info
cluster_state:ok
cluster_slots_assigned:16384
cluster_slots_ok:16384
cluster_slots_pfail:0
cluster_slots_fail:0
cluster_known_nodes:6
cluster_size:3
cluster_current_epoch:5
cluster_my_epoch:3
cluster_stats_messages_ping_sent:1084
cluster_stats_messages_pong_sent:1002
cluster_stats_messages_meet_sent:5
cluster_stats_messages_sent:2091
cluster_stats_messages_ping_received:1002
cluster_stats_messages_pong_received:1089
cluster_stats_messages_received:2091
[root@redis-01 ~]# redis-cli -h 10.10.10.21 -a 123456 --no-auth-warning cluster nodes
b1426dd91455998e04470338abe01be5292139e0 10.10.10.23:6379@16379 master - 0 1653141349000 4 connected 10923-16383
d658b014aa1b1741f795a7fd52c3d76fdfe4c01f 10.10.10.23:6380@16380 slave 37a9ea7fe27e44345a7e1a56a3417d41ec783a3b 0 1653141348000 3 connected
37a9ea7fe27e44345a7e1a56a3417d41ec783a3b 10.10.10.22:6379@16379 master - 0 1653141348711 3 connected 5462-10922
70a02a82092d8fd54c6e0b64090a6493ff83d4e3 10.10.10.21:6380@16380 slave b1426dd91455998e04470338abe01be5292139e0 0 1653141350000 4 connected
bcaa3e9f692ebd8775e67cf411f51c47c3842e31 10.10.10.22:6380@16380 slave 8b2d94e3bdb451883d4e32be15900262ebad1e70 0 1653141350784 1 connected
8b2d94e3bdb451883d4e32be15900262ebad1e70 10.10.10.21:6379@16379 myself,master - 0 1653141346000 1 connected 0-5461

[root@redis-01 ~]# redis-cli -a 123456 --cluster info 10.10.10.21:6379
Warning: Using a password with '-a' or '-u' option on the command line interface may not be safe.
10.10.10.21:6379 (8b2d94e3...) -> 0 keys | 5462 slots | 1 slaves.
10.10.10.23:6379 (b1426dd9...) -> 0 keys | 5461 slots | 1 slaves.
10.10.10.22:6379 (37a9ea7f...) -> 0 keys | 5461 slots | 1 slaves.
[OK] 0 keys in 3 masters.
0.00 keys per slot on average.
[root@redis-01 ~]# redis-cli -a 123456 --cluster info 10.10.10.22:6379
Warning: Using a password with '-a' or '-u' option on the command line interface may not be safe.
10.10.10.22:6379 (37a9ea7f...) -> 0 keys | 5461 slots | 1 slaves.
10.10.10.21:6379 (8b2d94e3...) -> 0 keys | 5462 slots | 1 slaves.
10.10.10.23:6379 (b1426dd9...) -> 0 keys | 5461 slots | 1 slaves.
[OK] 0 keys in 3 masters.
0.00 keys per slot on average.
[root@redis-01 ~]# redis-cli -a 123456 --cluster info 10.10.10.23:6379
Warning: Using a password with '-a' or '-u' option on the command line interface may not be safe.
10.10.10.23:6379 (b1426dd9...) -> 0 keys | 5461 slots | 1 slaves.
10.10.10.21:6379 (8b2d94e3...) -> 0 keys | 5462 slots | 1 slaves.
10.10.10.22:6379 (37a9ea7f...) -> 0 keys | 5461 slots | 1 slaves.
[OK] 0 keys in 3 masters.
0.00 keys per slot on average.
[root@redis-01 ~]# cat /apps/redis/data/nodes-6379.conf
b1426dd91455998e04470338abe01be5292139e0 10.10.10.23:6379@16379 master - 0 1653140816000 4 connected 10923-16383
d658b014aa1b1741f795a7fd52c3d76fdfe4c01f 10.10.10.23:6380@16380 slave 37a9ea7fe27e44345a7e1a56a3417d41ec783a3b 0 1653140817000 3 connected
37a9ea7fe27e44345a7e1a56a3417d41ec783a3b 10.10.10.22:6379@16379 master - 0 1653140817231 3 connected 5462-10922
70a02a82092d8fd54c6e0b64090a6493ff83d4e3 10.10.10.21:6380@16380 slave b1426dd91455998e04470338abe01be5292139e0 0 1653140817000 4 connected
bcaa3e9f692ebd8775e67cf411f51c47c3842e31 10.10.10.22:6380@16380 slave 8b2d94e3bdb451883d4e32be15900262ebad1e70 0 1653140818240 1 connected
8b2d94e3bdb451883d4e32be15900262ebad1e70 10.10.10.21:6379@16379 myself,master - 0 1653140816000 1 connected 0-5461
vars currentEpoch 5 lastVoteEpoch 0

##注：集群配置错误可以删除集群配置文件
[root@redis-01 ~]# ls -l /apps/redis/data/nodes-6379.conf
-rw-r--r-- 1 redis redis 793 May 21 21:46 /apps/redis/data/nodes-6379.conf
[root@redis-01 ~]# ls -l /apps/redis/data/nodes-6380.conf
-rw-r--r-- 1 redis redis 793 May 21 21:47 /apps/redis/data/nodes-6380.conf

2.2.2 一键配置redis cluster功能

## redis cluster 相关命令
[root@redis-01 ~]# redis-cli --cluster help
#命令redis-cli的选项 --cluster-replicas 1 表示每个master对应一个slave节点
[root@redis-01 ~]# redis-cli -a 123456 --cluster create 10.10.10.21:6379 10.10.10.22:6379 10.10.10.23:6379 10.10.10.22:6380 10.10.10.23:6380 10.10.10.21:6380  --cluster-replicas 1
Warning: Using a password with '-a' or '-u' option on the command line interface may not be safe.
>>> Performing hash slots allocation on 6 nodes...
Master[0] -> Slots 0 - 5460
Master[1] -> Slots 5461 - 10922
Master[2] -> Slots 10923 - 16383
Adding replica 10.10.10.22:6380 to 10.10.10.21:6379
Adding replica 10.10.10.23:6380 to 10.10.10.22:6379
Adding replica 10.10.10.21:6380 to 10.10.10.23:6379
M: 593941f699c32326bed085c11274e36c0bd8e0e7 10.10.10.21:6379   #带M的为master
   slots:[0-5460] (5461 slots) master                                         #当前master的槽位起始和结束位
M: 59b9e00713dbfbd1bfa12d24d4ea8e0db63ca057 10.10.10.22:6379
   slots:[5461-10922] (5462 slots) master
M: 762ebb9b4c8466ead6f27794cee1f8c839c286d1 10.10.10.23:6379
   slots:[10923-16383] (5461 slots) master
S: 8669956f1caa09486a48e136432b3c4e3ff42709 10.10.10.22:6380      #带S的slave
   replicates 593941f699c32326bed085c11274e36c0bd8e0e7
S: 30dcb0494250a6166f319158a6b41be4703dc95e 10.10.10.23:6380
   replicates 59b9e00713dbfbd1bfa12d24d4ea8e0db63ca057
S: e0f4b31d67eb52a790439d192c538d9b419fd73c 10.10.10.21:6380
   replicates 762ebb9b4c8466ead6f27794cee1f8c839c286d1
Can I set the above configuration? (type 'yes' to accept): yes   #输入yes自动创建集群
>>> Nodes configuration updated
>>> Assign a different config epoch to each node
>>> Sending CLUSTER MEET messages to join the cluster
Waiting for the cluster to join
.
>>> Performing Cluster Check (using node 10.10.10.21:6379)
M: 593941f699c32326bed085c11274e36c0bd8e0e7 10.10.10.21:6379
   slots:[0-5460] (5461 slots) master                       #已经分配的槽位
   1 additional replica(s)                                     #分配了一个slave
S: 8669956f1caa09486a48e136432b3c4e3ff42709 10.10.10.22:6380
   slots: (0 slots) slave                                         #slave没有分配槽位
   replicates 593941f699c32326bed085c11274e36c0bd8e0e7      #对应的master的10.10.10.21的ID
M: 762ebb9b4c8466ead6f27794cee1f8c839c286d1 10.10.10.23:6379
   slots:[10923-16383] (5461 slots) master
   1 additional replica(s)
S: e0f4b31d67eb52a790439d192c538d9b419fd73c 10.10.10.21:6380
   slots: (0 slots) slave
   replicates 762ebb9b4c8466ead6f27794cee1f8c839c286d1
S: 30dcb0494250a6166f319158a6b41be4703dc95e 10.10.10.23:6380
   slots: (0 slots) slave
   replicates 59b9e00713dbfbd1bfa12d24d4ea8e0db63ca057
M: 59b9e00713dbfbd1bfa12d24d4ea8e0db63ca057 10.10.10.22:6379
   slots:[5461-10922] (5462 slots) master
   1 additional replica(s)
[OK] All nodes agree about slots configuration.   #所有节点槽位分配完成
>>> Check for open slots...
>>> Check slots coverage...
[OK] All 16384 slots covered.       #所有槽位(16384个)分配完成

2.3 检查集群状态

##检查主从状态
[root@redis-01 ~]# redis-cli -h 10.10.10.21 -p 6379 -a 123456 -c info replication
Warning: Using a password with '-a' or '-u' option on the command line interface may not be safe.
# Replication
role:master
connected_slaves:1
slave0:ip=10.10.10.22,port=6380,state=online,offset=630,lag=1
master_failover_state:no-failover
master_replid:6cca6b5ceee6d89e0925753d3041f470e9d90a71
master_replid2:0000000000000000000000000000000000000000
master_repl_offset:630
second_repl_offset:-1
repl_backlog_active:1
repl_backlog_size:1048576
repl_backlog_first_byte_offset:1
repl_backlog_histlen:630
[root@redis-01 ~]# redis-cli -h 10.10.10.22 -p 6379 -a 123456 -c info replication
Warning: Using a password with '-a' or '-u' option on the command line interface may not be safe.
# Replication
role:master
connected_slaves:1
slave0:ip=10.10.10.23,port=6380,state=online,offset=630,lag=0
master_failover_state:no-failover
master_replid:41309e92aa4d40530db8e279a970af3197d5f251
master_replid2:0000000000000000000000000000000000000000
master_repl_offset:630
second_repl_offset:-1
repl_backlog_active:1
repl_backlog_size:1048576
repl_backlog_first_byte_offset:1
repl_backlog_histlen:630
[root@redis-01 ~]# redis-cli -h 10.10.10.23 -p 6379 -a 123456 -c info replication
Warning: Using a password with '-a' or '-u' option on the command line interface may not be safe.
# Replication
role:master
connected_slaves:1
slave0:ip=10.10.10.21,port=6380,state=online,offset=644,lag=0
master_failover_state:no-failover
master_replid:e8f0a0f0e259ba5491abc5001a61abd3893a5452
master_replid2:0000000000000000000000000000000000000000
master_repl_offset:644
second_repl_offset:-1
repl_backlog_active:1
repl_backlog_size:1048576
repl_backlog_first_byte_offset:1
repl_backlog_histlen:644
[root@redis-01 ~]# redis-cli -h 10.10.10.21 -p 6380 -a 123456 -c info replication
Warning: Using a password with '-a' or '-u' option on the command line interface may not be safe.
# Replication
role:slave
master_host:10.10.10.23
master_port:6379
master_link_status:up
master_last_io_seconds_ago:4
master_sync_in_progress:0
slave_repl_offset:658
slave_priority:100
slave_read_only:1
replica_announced:1
connected_slaves:0
master_failover_state:no-failover
master_replid:e8f0a0f0e259ba5491abc5001a61abd3893a5452
master_replid2:0000000000000000000000000000000000000000
master_repl_offset:658
second_repl_offset:-1
repl_backlog_active:1
repl_backlog_size:1048576
repl_backlog_first_byte_offset:1
repl_backlog_histlen:658
[root@redis-01 ~]# redis-cli -h 10.10.10.22 -p 6380 -a 123456 -c info replication
Warning: Using a password with '-a' or '-u' option on the command line interface may not be safe.
# Replication
role:slave
master_host:10.10.10.21
master_port:6379
master_link_status:up
master_last_io_seconds_ago:3
master_sync_in_progress:0
slave_repl_offset:714
slave_priority:100
slave_read_only:1
replica_announced:1
connected_slaves:0
master_failover_state:no-failover
master_replid:6cca6b5ceee6d89e0925753d3041f470e9d90a71
master_replid2:0000000000000000000000000000000000000000
master_repl_offset:714
second_repl_offset:-1
repl_backlog_active:1
repl_backlog_size:1048576
repl_backlog_first_byte_offset:1
repl_backlog_histlen:714
[root@redis-01 ~]# redis-cli -h 10.10.10.23 -p 6380 -a 123456 -c info replication
Warning: Using a password with '-a' or '-u' option on the command line interface may not be safe.
# Replication
role:slave
master_host:10.10.10.22
master_port:6379
master_link_status:up
master_last_io_seconds_ago:8
master_sync_in_progress:0
slave_repl_offset:714
slave_priority:100
slave_read_only:1
replica_announced:1
connected_slaves:0
master_failover_state:no-failover
master_replid:41309e92aa4d40530db8e279a970af3197d5f251
master_replid2:0000000000000000000000000000000000000000
master_repl_offset:714
second_repl_offset:-1
repl_backlog_active:1
repl_backlog_size:1048576
repl_backlog_first_byte_offset:1
repl_backlog_histlen:714

##查看集群node对应关系
[root@redis-01 ~]# redis-cli -a 123456 cluster nodes
Warning: Using a password with '-a' or '-u' option on the command line interface may not be safe.
8669956f1caa09486a48e136432b3c4e3ff42709 10.10.10.22:6380@16380 slave 593941f699c32326bed085c11274e36c0bd8e0e7 0 1653143464088 1 connected
762ebb9b4c8466ead6f27794cee1f8c839c286d1 10.10.10.23:6379@16379 master - 0 1653143463041 3 connected 10923-16383
e0f4b31d67eb52a790439d192c538d9b419fd73c 10.10.10.21:6380@16380 slave 762ebb9b4c8466ead6f27794cee1f8c839c286d1 0 1653143465151 3 connected
30dcb0494250a6166f319158a6b41be4703dc95e 10.10.10.23:6380@16380 slave 59b9e00713dbfbd1bfa12d24d4ea8e0db63ca057 0 1653143462000 2 connected
59b9e00713dbfbd1bfa12d24d4ea8e0db63ca057 10.10.10.22:6379@16379 master - 0 1653143461993 2 connected 5461-10922
593941f699c32326bed085c11274e36c0bd8e0e7 10.10.10.21:6379@16379 myself,master - 0 1653143462000 1 connected 0-5460
##检查集群状态
[root@redis-01 ~]# redis-cli -a 123456 cluster info
Warning: Using a password with '-a' or '-u' option on the command line interface may not be safe.
cluster_state:ok
cluster_slots_assigned:16384
cluster_slots_ok:16384
cluster_slots_pfail:0
cluster_slots_fail:0
cluster_known_nodes:6
cluster_size:3
cluster_current_epoch:6
cluster_my_epoch:1
cluster_stats_messages_ping_sent:702
cluster_stats_messages_pong_sent:700
cluster_stats_messages_sent:1402
cluster_stats_messages_ping_received:695
cluster_stats_messages_pong_received:702
cluster_stats_messages_meet_received:5
cluster_stats_messages_received:1402

#查看任意节点的集群状态
[root@redis-01 ~]# redis-cli -a 123456 --cluster info 10.10.10.21:6379
Warning: Using a password with '-a' or '-u' option on the command line interface may not be safe.
10.10.10.21:6379 (593941f6...) -> 0 keys | 5461 slots | 1 slaves.
10.10.10.23:6379 (762ebb9b...) -> 0 keys | 5461 slots | 1 slaves.
10.10.10.22:6379 (59b9e007...) -> 0 keys | 5462 slots | 1 slaves.
[OK] 0 keys in 3 masters.
0.00 keys per slot on average.
[root@redis-01 ~]# redis-cli -a 123456 --cluster info 10.10.10.22:6379
Warning: Using a password with '-a' or '-u' option on the command line interface may not be safe.
10.10.10.22:6379 (59b9e007...) -> 0 keys | 5462 slots | 1 slaves.
10.10.10.23:6379 (762ebb9b...) -> 0 keys | 5461 slots | 1 slaves.
10.10.10.21:6379 (593941f6...) -> 0 keys | 5461 slots | 1 slaves.
[OK] 0 keys in 3 masters.
0.00 keys per slot on average.
[root@redis-01 ~]# redis-cli -a 123456 --cluster info 10.10.10.23:6379
Warning: Using a password with '-a' or '-u' option on the command line interface may not be safe.
10.10.10.23:6379 (762ebb9b...) -> 0 keys | 5461 slots | 1 slaves.
10.10.10.22:6379 (59b9e007...) -> 0 keys | 5462 slots | 1 slaves.
10.10.10.21:6379 (593941f6...) -> 0 keys | 5461 slots | 1 slaves.
[OK] 0 keys in 3 masters.
0.00 keys per slot on average.
##验证数据的写入和查询
[root@redis-01 ~]# redis-cli -c -h 10.10.10.23 -p 6379 -a 123456 --no-auth-warning
10.10.10.23:6379> set redis 6.2.4
-> Redirected to slot [1151] located at 10.10.10.21:6379
OK
10.10.10.21:6379> get redis
"6.2.4"

2.4 模拟master 故障，对应的slave节点自动提升为新master

##模拟master 10.10.23:6379故障  slave 10.10.10.21:6380 自动提升为master
[root@redis-03 ~]# systemctl stop redis_6379
[root@redis-01 ~]# tail /apps/redis/log/redis_6380.log
55328:S 21 May 2022 22:50:29.169 * FAIL message received from 59b9e00713dbfbd1bfa12d24d4ea8e0db63ca057 about 762ebb9b4c8466ead6f27794cee1f8c839c286d1
55328:S 21 May 2022 22:50:29.269 # Start of election delayed for 940 milliseconds (rank #0, offset 2366).
55328:S 21 May 2022 22:50:29.373 * Connecting to MASTER 10.10.10.23:6379
55328:S 21 May 2022 22:50:29.374 * MASTER <-> REPLICA sync started
55328:S 21 May 2022 22:50:29.375 # Error condition on socket for SYNC: Connection refused
55328:S 21 May 2022 22:50:30.314 # Starting a failover election for epoch 7.
55328:S 21 May 2022 22:50:30.325 # Failover election won: I'm the new masteru
55328:S 21 May 2022 22:50:30.325 # configEpoch set to 7 after successful failover
55328:M 21 May 2022 22:50:30.325 * Discarding previously cached master state.
55328:M 21 May 2022 22:50:30.325 # Setting secondary replication ID to e8f0a0f0e259ba5491abc5001a61abd3893a5452, valid up to offset: 2367. New replication ID is 9300efee569ebc91d8cab02844ef33a55175981b
[root@redis-01 ~]# redis-cli -h 10.10.10.21 -p 6379 -a 123456 --no-auth-warning
10.10.10.21:6379> cluster info
cluster_state:ok
cluster_slots_assigned:16384
cluster_slots_ok:16384
cluster_slots_pfail:0
cluster_slots_fail:0
cluster_known_nodes:6
cluster_size:3
cluster_current_epoch:7
cluster_my_epoch:1
cluster_stats_messages_ping_sent:1948
cluster_stats_messages_pong_sent:1901
cluster_stats_messages_fail_sent:4
cluster_stats_messages_auth-ack_sent:1
cluster_stats_messages_sent:3854
cluster_stats_messages_ping_received:1896
cluster_stats_messages_pong_received:1948
cluster_stats_messages_meet_received:5
cluster_stats_messages_fail_received:1
cluster_stats_messages_auth-req_received:1
cluster_stats_messages_received:3851
10.10.10.21:6379> cluster nodes
8669956f1caa09486a48e136432b3c4e3ff42709 10.10.10.22:6380@16380 slave 593941f699c32326bed085c11274e36c0bd8e0e7 0 1653144912000 1 connected
762ebb9b4c8466ead6f27794cee1f8c839c286d1 10.10.10.23:6379@16379 master,fail - 1653144610329 1653144605064 3 disconnected
e0f4b31d67eb52a790439d192c538d9b419fd73c 10.10.10.21:6380@16380 master - 0 1653144911000 7 connected 10923-16383
30dcb0494250a6166f319158a6b41be4703dc95e 10.10.10.23:6380@16380 slave 59b9e00713dbfbd1bfa12d24d4ea8e0db63ca057 0 1653144913624 2 connected
59b9e00713dbfbd1bfa12d24d4ea8e0db63ca057 10.10.10.22:6379@16379 master - 0 1653144910516 2 connected 5461-10922
593941f699c32326bed085c11274e36c0bd8e0e7 10.10.10.21:6379@16379 myself,master - 0 1653144910000 1 connected 0-5460
[root@redis-02 ~]# redis-cli -a 123456 --cluster info 10.10.10.21:6380
Warning: Using a password with '-a' or '-u' option on the command line interface may not be safe.
Could not connect to Redis at 10.10.10.23:6379: Connection refused
10.10.10.21:6380 (e0f4b31d...) -> 0 keys | 5461 slots | 0 slaves.
10.10.10.22:6379 (59b9e007...) -> 0 keys | 5462 slots | 1 slaves.
10.10.10.21:6379 (593941f6...) -> 1 keys | 5461 slots | 1 slaves.
[OK] 1 keys in 3 masters.
0.00 keys per slot on average.
[root@redis-02 ~]# redis-cli -a 123456 --cluster info 10.10.10.22:6380
Warning: Using a password with '-a' or '-u' option on the command line interface may not be safe.
Could not connect to Redis at 10.10.10.23:6379: Connection refused
10.10.10.22:6379 (59b9e007...) -> 0 keys | 5462 slots | 1 slaves.
10.10.10.21:6380 (e0f4b31d...) -> 0 keys | 5461 slots | 0 slaves.
10.10.10.21:6379 (593941f6...) -> 1 keys | 5461 slots | 1 slaves.
[OK] 1 keys in 3 masters.
0.00 keys per slot on average.
[root@redis-02 ~]# redis-cli -a 123456 --cluster info 10.10.10.23:6380
Warning: Using a password with '-a' or '-u' option on the command line interface may not be safe.
Could not connect to Redis at 10.10.10.23:6379: Connection refused
10.10.10.21:6380 (e0f4b31d...) -> 0 keys | 5461 slots | 0 slaves.
10.10.10.22:6379 (59b9e007...) -> 0 keys | 5462 slots | 1 slaves.
10.10.10.21:6379 (593941f6...) -> 1 keys | 5461 slots | 1 slaves.
[OK] 1 keys in 3 masters.
0.00 keys per slot on average.
##重新恢复10.10.10.23:6379
[root@redis-03 ~]# systemctl start redis_6379
[root@redis-01 ~]# tail -f /apps/redis/log/redis_6379.log
55327:M 21 May 2022 22:20:40.971 * Starting BGSAVE for SYNC with target: disk
55327:M 21 May 2022 22:20:40.972 * Background saving started by pid 55367
55367:C 21 May 2022 22:20:40.980 * DB saved on disk
55367:C 21 May 2022 22:20:40.984 * RDB: 0 MB of memory used by copy-on-write
55327:M 21 May 2022 22:20:41.044 * Background saving terminated with success
55327:M 21 May 2022 22:20:41.044 * Synchronization with replica 10.10.10.22:6380 succeeded
55327:M 21 May 2022 22:20:43.848 # Cluster state changed: ok
55327:M 21 May 2022 22:50:29.168 * Marking node 762ebb9b4c8466ead6f27794cee1f8c839c286d1 as failing (quorum reached).
55327:M 21 May 2022 22:50:30.318 # Failover auth granted to e0f4b31d67eb52a790439d192c538d9b419fd73c for epoch 7
55327:M 21 May 2022 23:00:27.657 * Clear FAIL state for node 762ebb9b4c8466ead6f27794cee1f8c839c286d1: master without slots is reachable again.

10.10.10.21:6379> cluster nodes
8669956f1caa09486a48e136432b3c4e3ff42709 10.10.10.22:6380@16380 slave 593941f699c32326bed085c11274e36c0bd8e0e7 0 1653145442573 1 connected
762ebb9b4c8466ead6f27794cee1f8c839c286d1 10.10.10.23:6379@16379 slave e0f4b31d67eb52a790439d192c538d9b419fd73c 0 1653145439000 7 connected
e0f4b31d67eb52a790439d192c538d9b419fd73c 10.10.10.21:6380@16380 master - 0 1653145441536 7 connected 10923-16383
30dcb0494250a6166f319158a6b41be4703dc95e 10.10.10.23:6380@16380 slave 59b9e00713dbfbd1bfa12d24d4ea8e0db63ca057 0 1653145439446 2 connected
59b9e00713dbfbd1bfa12d24d4ea8e0db63ca057 10.10.10.22:6379@16379 master - 0 1653145441000 2 connected 5461-10922
593941f699c32326bed085c11274e36c0bd8e0e7 10.10.10.21:6379@16379 myself,master - 0 1653145441000 1 connected 0-5460
10.10.10.21:6379>

[root@redis-02 ~]# redis-cli -a 123456 --cluster info 10.10.10.23:6380
Warning: Using a password with '-a' or '-u' option on the command line interface may not be safe.
10.10.10.21:6380 (e0f4b31d...) -> 0 keys | 5461 slots | 1 slaves.
10.10.10.22:6379 (59b9e007...) -> 0 keys | 5462 slots | 1 slaves.
10.10.10.21:6379 (593941f6...) -> 1 keys | 5461 slots | 1 slaves.
[OK] 1 keys in 3 masters.
0.00 keys per slot on average.
[root@redis-03 ~]# redis-cli -h 10.10.10.23 -p 6379 -a 123456 info replication
Warning: Using a password with '-a' or '-u' option on the command line interface may not be safe.
# Replication
role:slave
master_host:10.10.10.21
master_port:6380
master_link_status:up
master_last_io_seconds_ago:4
master_sync_in_progress:0
slave_repl_offset:2856
slave_priority:100
slave_read_only:1
replica_announced:1
connected_slaves:0
master_failover_state:no-failover
master_replid:9300efee569ebc91d8cab02844ef33a55175981b
master_replid2:0000000000000000000000000000000000000000
master_repl_offset:2856
second_repl_offset:-1
repl_backlog_active:1
repl_backlog_size:1048576
repl_backlog_first_byte_offset:2367
repl_backlog_histlen:490
[root@redis-03 ~]# redis-cli -h 10.10.10.21 -p 6380 -a 123456 info replication
Warning: Using a password with '-a' or '-u' option on the command line interface may not be safe.
# Replication
role:master
connected_slaves:1
slave0:ip=10.10.10.23,port=6379,state=online,offset=2870,lag=0
master_failover_state:no-failover
master_replid:9300efee569ebc91d8cab02844ef33a55175981b
master_replid2:e8f0a0f0e259ba5491abc5001a61abd3893a5452
master_repl_offset:2870
second_repl_offset:2367
repl_backlog_active:1
repl_backlog_size:1048576
repl_backlog_first_byte_offset:1
repl_backlog_histlen:2870