摘要:
说明如何快速用spotahome-redis-operator部署哨兵加主从
github地址:
部署流程:
一. 部署CRD
kubectl apply -f charts/redisoperator/crds/databases.spotahome.com_redisfailovers.yaml[root@node-201 redis-operator]# kubectl get crd | grep fail
redisfailovers.databases.spotahome.com 2022-04-13T09:35:01Z
二. 部署operator的pod
kubectl apply -f example/operator[root@node-201 redis-operator]# kubectl get pods | grep operator
redisoperator-64d6d5455b-6nqds 1/1 Running 0 44m
三. 部署最简单的哨兵+主从
kubectl apply -f example/redisfailover/basic.yaml
[root@node-201 redis-operator]# kubectl get pods -o wide
NAME READY STATUS RESTARTS AGE IP NODE NOMINATED NODE READINESS GATES
redisoperator-64d6d5455b-6nqds 1/1 Running 0 47m 10.168.0.61 node-201 <none> <none>
rfr-redisfailover-0 0/1 Running 0 70s 10.168.0.104 node-201 <none> <none>
rfr-redisfailover-1 0/1 Running 0 70s 10.168.0.103 node-201 <none> <none>
rfr-redisfailover-2 0/1 Running 0 70s 10.168.0.108 node-201 <none> <none>
rfs-redisfailover-857f74578f-2xlf8 0/1 Running 0 70s 10.168.0.105 node-201 <none> <none>
rfs-redisfailover-857f74578f-wxw6l 0/1 Running 0 70s 10.168.0.106 node-201 <none> <none>
rfs-redisfailover-857f74578f-xx4q7 0/1 Running 0 70s 10.168.0.107 node-201 <none> <none>
四. 校验主从关系
[root@node-201 redis-operator]# kubectl exec rfr-redisfailover-0 -it -- redis-cli info Replication
# Replication
role:master
connected_slaves:2
slave0:ip=10.168.0.103,port=6379,state=online,offset=1274,lag=1
slave1:ip=10.168.0.108,port=6379,state=online,offset=1274,lag=1
master_failover_state:no-failover
master_replid:cc5947aea24246662d2e119da3699382d8d95f07
master_replid2:0000000000000000000000000000000000000000
master_repl_offset:1274
second_repl_offset:-1
repl_backlog_active:1
repl_backlog_size:1048576
repl_backlog_first_byte_offset:1
repl_backlog_histlen:1274
[root@node-201 redis-operator]# kubectl exec rfr-redisfailover-1 -it -- redis-cli info Replication
# Replication
role:slave
master_host:10.168.0.104
master_port:6379
master_link_status:up
master_last_io_seconds_ago:1
master_sync_in_progress:0
slave_read_repl_offset:4638
slave_repl_offset:4638
slave_priority:100
slave_read_only:1
replica_announced:1
connected_slaves:0
master_failover_state:no-failover
master_replid:cc5947aea24246662d2e119da3699382d8d95f07
master_replid2:0000000000000000000000000000000000000000
master_repl_offset:4638
second_repl_offset:-1
repl_backlog_active:1
repl_backlog_size:1048576
repl_backlog_first_byte_offset:1
repl_backlog_histlen:4638
[root@node-201 redis-operator]# kubectl exec rfr-redisfailover-2 -it -- redis-cli info Replication
# Replication
role:slave
master_host:10.168.0.104
master_port:6379
master_link_status:up
master_last_io_seconds_ago:1
master_sync_in_progress:0
slave_read_repl_offset:8002
slave_repl_offset:8002
slave_priority:100
slave_read_only:1
replica_announced:1
connected_slaves:0
master_failover_state:no-failover
master_replid:cc5947aea24246662d2e119da3699382d8d95f07
master_replid2:0000000000000000000000000000000000000000
master_repl_offset:8002
second_repl_offset:-1
repl_backlog_active:1
repl_backlog_size:1048576
repl_backlog_first_byte_offset:1
repl_backlog_histlen:8002
[root@node-201 redis-operator]# kubectl exec rfs-redisfailover-857f74578f-2xlf8 -it -- redis-cli -p 26379 info Sentinel
# Sentinel
sentinel_masters:1
sentinel_tilt:0
sentinel_running_scripts:0
sentinel_scripts_queue_length:0
sentinel_simulate_failure_flags:0
master0:name=mymaster,status=ok,address=10.168.0.104:6379,slaves=2,sentinels=3
编译调试operator
一. 编译operator
go build -gcflags=all="-N -l" -o ctr ./cmd/redisoperator/main.go
二. 直接在主机上而非pod内运行
export KUBERNETES_SERVICE_HOST="127.0.0.1"
export KUBERNETES_SERVICE_PORT="6443"
./ctr -development=true
三. gdb 调试
[root@node-201 redis-operator]# ps -ef | grep ctr
root 979 1 0 18:24 ? 00:00:00 /usr/sbin/sshd -D -oCiphers=aes256-gcm@openssh.com,chacha20-poly1305@openssh.com,aes256-ctr,aes256-cbc,aes128-gcm@openssh.com,aes128-ctr,aes128-cbc -oMACs=hmac-sha2-256-etm@openssh.com,hmac-sha1-etm@openssh.com,umac-128-etm@openssh.com,hmac-sha2-512-etm@openssh.com,hmac-sha2-256,hmac-sha1,umac-128@openssh.com,hmac-sha2-512 -oGSSAPIKexAlgorithms=gss-curve25519-sha256-,gss-nistp256-sha256-,gss-group14-sha256-,gss-group16-sha512-,gss-gex-sha1-,gss-group14-sha1- -oKexAlgorithms=curve25519-sha256,curve25519-sha256@libssh.org,ecdh-sha2-nistp256,ecdh-sha2-nistp384,ecdh-sha2-nistp521,diffie-hellman-group-exchange-sha256,diffie-hellman-group14-sha256,diffie-hellman-group16-sha512,diffie-hellman-group18-sha512,diffie-hellman-group-exchange-sha1,diffie-hellman-group14-sha1 -oHostKeyAlgorithms=ecdsa-sha2-nistp256,ecdsa-sha2-nistp256-cert-v01@openssh.com,ecdsa-sha2-nistp384,ecdsa-sha2-nistp384-cert-v01@openssh.com,ecdsa-sha2-nistp521,ecdsa-sha2-nistp521-cert-v01@openssh.com,ssh-ed25519,ssh-ed25519-cert-v01@openssh.com,rsa-sha2-256,rsa-sha2-256-cert-v01@openssh.com,rsa-sha2-512,rsa-sha2-512-cert-v01@openssh.com,ssh-rsa,ssh-rsa-cert-v01@openssh.com -oPubkeyAcceptedKeyTypes=ecdsa-sha2-nistp256,ecdsa-sha2-nistp256-cert-v01@openssh.com,ecdsa-sha2-nistp384,ecdsa-sha2-nistp384-cert-v01@openssh.com,ecdsa-sha2-nistp521,ecdsa-sha2-nistp521-cert-v01@openssh.com,ssh-ed25519,ssh-ed25519-cert-v01@openssh.com,rsa-sha2-256,rsa-sha2-256-cert-v01@openssh.com,rsa-sha2-512,rsa-sha2-512-cert-v01@openssh.com,ssh-rsa,ssh-rsa-cert-v01@openssh.com -oCASignatureAlgorithms=ecdsa-sha2-nistp256,ecdsa-sha2-nistp384,ecdsa-sha2-nistp521,ssh-ed25519,rsa-sha2-256,rsa-sha2-512,ssh-rsa
root 40809 8435 0 18:40 pts/0 00:00:00 bash run_ctr.sh
root 40810 40809 0 18:40 pts/0 00:00:02 ./ctr -development=true
root 71376 34652 0 18:57 pts/3 00:00:00 grep --color=auto ctr
[root@node-201 redis-operator]# gdb -p 40810
GNU gdb (GDB) Red Hat Enterprise Linux 8.2-18.el8
Copyright (C) 2018 Free Software Foundation, Inc.
License GPLv3+: GNU GPL version 3 or later <http://gnu.org/licenses/gpl.html>
This is free software: you are free to change and redistribute it.
There is NO WARRANTY, to the extent permitted by law.
Type "show copying" and "show warranty" for details.
This GDB was configured as "x86_64-redhat-linux-gnu".
Type "show configuration" for configuration details.
For bug reporting instructions, please see:
<http://www.gnu.org/software/gdb/bugs/>.
Find the GDB manual and other documentation resources online at:
<http://www.gnu.org/software/gdb/documentation/>.
For help, type "help".
Type "apropos word" to search for commands related to "word".
Attaching to process 40810
[New LWP 40811]
[New LWP 40812]
[New LWP 40813]
[New LWP 40814]
[New LWP 40815]
[New LWP 40816]
[New LWP 40817]
[New LWP 40818]
[New LWP 59999]
[Thread debugging using libthread_db enabled]
Using host libthread_db library "/lib64/libthread_db.so.1".
runtime.futex () at /usr/lib/golang/src/runtime/sys_linux_amd64.s:520
520 MOVL AX, ret+40(FP)
warning: File "/usr/lib/golang/src/runtime/runtime-gdb.py" auto-loading has been declined by your `auto-load safe-path' set to "$debugdir:$datadir/auto-load:/usr/lib/golang/src/pkg/runtime/runtime-gdb.py".
To enable execution of this file add
add-auto-load-safe-path /usr/lib/golang/src/runtime/runtime-gdb.py
line to your configuration file "/root/.gdbinit".
To completely disable this security protection add
set auto-load safe-path /
line to your configuration file "/root/.gdbinit".
For more information about this security protection see the
"Auto-loading safe path" section in the GDB manual. E.g., run from the shell:
info "(gdb)Auto-loading safe path"
Missing separate debuginfos, use: yum debuginfo-install glibc-2.28-196.el8.x86_64 openssl-libs-1.1.1k-6.el8.x86_64 zlib-1.2.11-17.el8.x86_64
(gdb) b checker.go:90
Breakpoint 1 at 0x1c3a6a2: file /root/work/redis-operator/operator/redisfailover/checker.go, line 90.
(gdb) c
Continuing.
[Switching to Thread 0x7f6afc9e6700 (LWP 40817)]
Thread 8 "ctr" hit Breakpoint 1, github.com/spotahome/redis-operator/operator/redisfailover.(*RedisFailoverHandler).CheckAndHeal (r=0xc0004aa100, rf=0xc0000c1680, ~r1=...)
at /root/work/redis-operator/operator/redisfailover/checker.go:90
90 if rf.Bootstrapping() {
(gdb) bt
#0 github.com/spotahome/redis-operator/operator/redisfailover.(*RedisFailoverHandler).CheckAndHeal (r=0xc0004aa100, rf=0xc0000c1680, ~r1=...) at /root/work/redis-operator/operator/redisfailover/checker.go:90
#1 0x0000000001c3db7a in github.com/spotahome/redis-operator/operator/redisfailover.(*RedisFailoverHandler).Handle (r=0xc0004aa100, obj=..., ~r2=...) at /root/work/redis-operator/operator/redisfailover/handler.go:79
#2 0x0000000001b171f2 in github.com/spotahome/kooper/v2/controller.newIndexerProcessor.func1 (ctx=..., key=..., ~r2=...) at /root/work/redis-operator/vendor/github.com/spotahome/kooper/v2/controller/processor.go:39
#3 0x0000000001b16eb1 in github.com/spotahome/kooper/v2/controller.processorFunc.Process (p={void (context.Context, string, error *)} 0xc0004eb968, ctx=..., key=..., ~r2=...)
at /root/work/redis-operator/vendor/github.com/spotahome/kooper/v2/controller/processor.go:22
#4 0x0000000001b17d27 in github.com/spotahome/kooper/v2/controller.newMetricsProcessor.func1 (ctx=..., key=..., err=...) at /root/work/redis-operator/vendor/github.com/spotahome/kooper/v2/controller/processor.go:74
#5 0x0000000001b16eb1 in github.com/spotahome/kooper/v2/controller.processorFunc.Process (p={void (context.Context, string, error *)} 0xc0004ebb40, ctx=..., key=..., ~r2=...)
at /root/work/redis-operator/vendor/github.com/spotahome/kooper/v2/controller/processor.go:22
#6 0x0000000001b16976 in github.com/spotahome/kooper/v2/controller.(*generic).processNextJob (g=0xc0001e80f0, ~r0=false)
at /root/work/redis-operator/vendor/github.com/spotahome/kooper/v2/controller/controller.go:302
#7 0x0000000001b16667 in github.com/spotahome/kooper/v2/controller.(*generic).runWorker (g=0xc0001e80f0) at /root/work/redis-operator/vendor/github.com/spotahome/kooper/v2/controller/controller.go:279
#8 0x0000000001b19a4b in github.com/spotahome/kooper/v2/controller.(*generic).runWorker-fm () at /root/work/redis-operator/vendor/github.com/spotahome/kooper/v2/controller/controller.go:276
#9 0x0000000000ce94bc in k8s.io/apimachinery/pkg/util/wait.BackoffUntil.func1 (f={void (void)} 0xc0004ebe38) at /root/work/redis-operator/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:155
#10 0x0000000000ce92c9 in k8s.io/apimachinery/pkg/util/wait.BackoffUntil (f={void (void)} 0xc0004ebef0, backoff=..., sliding=true, stopCh=0x0)
at /root/work/redis-operator/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:156
#11 0x0000000000ce91a5 in k8s.io/apimachinery/pkg/util/wait.JitterUntil (f={void (void)} 0xc0004ebf48, period=1000000000, jitterFactor=0, sliding=true, stopCh=0x0)
at /root/work/redis-operator/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:133
#12 0x0000000000ce90d3 in k8s.io/apimachinery/pkg/util/wait.Until (f={void (void)} 0xc0004ebf80, period=1000000000, stopCh=0x0) at /root/work/redis-operator/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:90
#13 0x0000000001b164b5 in github.com/spotahome/kooper/v2/controller.(*generic).run.func1 () at /root/work/redis-operator/vendor/github.com/spotahome/kooper/v2/controller/controller.go:263
#14 0x000000000046f861 in runtime.goexit () at /usr/lib/golang/src/runtime/asm_amd64.s:1581
#15 0x0000000000000000 in ?? ()
四. 启动日志:
[root@node-201 redis-operator]# bash run_ctr.sh
INFO[0000] Listening on :9710 for metrics exposure src="asm_amd64.s:1581"
INFO[0000] starting controller controller-id=redisfailover operator=redisfailover service=kooper.controller src="controller.go:233"
INFO[0000] configMap updated configMap=rfs-redisfailover namespace=default service=k8s.configMap src="configmap.go:78"
INFO[0000] configMap updated configMap=rfr-s-redisfailover namespace=default service=k8s.configMap src="configmap.go:78"
INFO[0000] configMap updated configMap=rfr-readiness-redisfailover namespace=default service=k8s.configMap src="configmap.go:78"
INFO[0000] configMap updated configMap=rfr-redisfailover namespace=default service=k8s.configMap src="configmap.go:78"
INFO[0000] podDisruptionBudget updated namespace=default podDisruptionBudget=rfr-redisfailover service=k8s.podDisruptionBudget src="poddisruptionbudget.go:79"
INFO[0000] statefulSet updated namespace=default service=k8s.statefulSet src="statefulset.go:102" statefulSet=rfr-redisfailover
INFO[0000] podDisruptionBudget updated namespace=default podDisruptionBudget=rfs-redisfailover service=k8s.podDisruptionBudget src="poddisruptionbudget.go:79"
INFO[0000] deployment updated deployment=rfs-redisfailover namespace=default service=k8s.deployment src="deployment.go:102"
W0413 18:42:53.823638 40810 reflector.go:441] github.com/spotahome/kooper/v2/controller/controller.go:252: watch of <unspecified> ended with: an error on the server ("unable to decode an event from the watch stream: http2: client connection lost") has prevented the request from succeeding
INFO[0132] configMap updated configMap=rfs-redisfailover namespace=default service=k8s.configMap src="configmap.go:78"
INFO[0132] configMap updated configMap=rfr-s-redisfailover namespace=default service=k8s.configMap src="configmap.go:78"
INFO[0132] configMap updated configMap=rfr-readiness-redisfailover namespace=default service=k8s.configMap src="configmap.go:78"
INFO[0132] configMap updated configMap=rfr-redisfailover namespace=default service=k8s.configMap src="configmap.go:78"
INFO[0132] podDisruptionBudget updated namespace=default podDisruptionBudget=rfr-redisfailover service=k8s.podDisruptionBudget src="poddisruptionbudget.go:79"
INFO[0132] statefulSet updated namespace=default service=k8s.statefulSet src="statefulset.go:102" statefulSet=rfr-redisfailover
INFO[0132] podDisruptionBudget updated namespace=default podDisruptionBudget=rfs-redisfailover service=k8s.podDisruptionBudget src="poddisruptionbudget.go:79"
INFO[0132] deployment updated deployment=rfs-redisfailover namespace=default service=k8s.deployment src="deployment.go:102"
INFO[0260] configMap updated configMap=rfs-redisfailover namespace=default service=k8s.configMap src="configmap.go:78"
INFO[0260] configMap updated configMap=rfr-s-redisfailover namespace=default service=k8s.configMap src="configmap.go:78"
INFO[0260] configMap updated configMap=rfr-readiness-redisfailover namespace=default service=k8s.configMap src="configmap.go:78"
INFO[0260] configMap updated configMap=rfr-redisfailover namespace=default service=k8s.configMap src="configmap.go:78"
INFO[0260] podDisruptionBudget updated namespace=default podDisruptionBudget=rfr-redisfailover service=k8s.podDisruptionBudget src="poddisruptionbudget.go:79"
INFO[0260] statefulSet updated namespace=default service=k8s.statefulSet src="statefulset.go:102" statefulSet=rfr-redisfailover
INFO[0260] podDisruptionBudget updated namespace=default podDisruptionBudget=rfs-redisfailover service=k8s.podDisruptionBudget src="poddisruptionbudget.go:79"
INFO[0260] deployment updated deployment=rfs-redisfailover namespace=default service=k8s.deployment src="deployment.go:102"
INFO[0290] configMap updated configMap=rfs-redisfailover namespace=default service=k8s.configMap src="configmap.go:78"
INFO[0290] configMap updated configMap=rfr-s-redisfailover namespace=default service=k8s.configMap src="configmap.go:78"
