mongodb分片架构图:
一.准备环境:
1.centos7 环境
2.安装ansible环境
3.mongodb二进制文件:可以先用yum安装拿取需要mongodb二进制文件
mongod, mongodump,mongoexport, mongofiles,mongoimport,mongorestore,mongos,mongosh,mongostat
4.mongodb安装主机,作者用192.168.126.128,192.168.126.129,192.168.126.130作为mongodb安装环境
注意: 先启动config→mongos→shard
二.规划:
1.变量规划:
安装路径:mongodb_dir: /kingdee/mongodb
运行用户:mongodb_start_user: yzj
用户名称:mongodbuser: yzjuser
用户密码:mongodbpass: Mytest@@@@@@1203
实例名称:mongodb_instance: datacore
集群shards【可以根据需求增加或减少】:
{"name": "shard1","ip": "192.168.126.128,192.168.126.129,192.168.126.130","port":"27017","memory":"2"}
{"name": "shard2","ip": "192.168.126.128,192.168.126.129,192.168.126.130","port":"27018","memory":"2"}
{"name": "shard3","ip": "192.168.126.128,192.168.126.129,192.168.126.130","port":"27019","memory":"2"}
配置config:
{"name": "configs","ip":"192.168.126.128,192.168.126.129,192.168.126.130","port":"20001","memory":"2"}
路由mongos:
{"name": "mongos","ip":"192.168.126.128,192.168.126.129,192.168.126.130","port":"20000","memory":"1"}
2.mongodb目录规划:
数据目录:/kingdee/mongodb/datacore/data
执行文件:/kingdee/mongodb/datacore/bin
日志文件:/kingdee/mongodb/datacore/logs
三.编写ansible-playbook
1.创建ansible-playbook目录
mkdir -p roles/mongodb_shards/{files,tasks,templates,vars}
2.模板文件配置
cd roles/mongodb_shards/templates
#config认证模板
vim config_mongodb_auth.conf.j2
logpath= /kingdee/mongodb/{{mongodb_instance}}/logs/mongod_{{item.port}}.log
dbpath=/kingdee/mongodb/{{mongodb_instance}}/data/{{item.name}}
fork=true
pidfilepath=/kingdee/mongodb/{{mongodb_instance}}/pid/mongod_{{item.port}}.pid
port={{item.port}}
bind_ip=0.0.0.0
replSet={{item.name}}
wiredTigerCacheSizeGB={{item.memory}}
configsvr=true
keyFile={{mongodb_dir}}/{{mongodb_instance}}/bin/keyFile
timeZoneInfo=/usr/share/zoneinfo
auth=true
#config无需认证模板
vim config_mongodb_unauth.conf.j2
logpath= /kingdee/mongodb/{{mongodb_instance}}/logs/config_mongod_{{item.port}}.log
dbpath=/kingdee/mongodb/{{mongodb_instance}}/data/{{item.name}}
fork=true
pidfilepath=/kingdee/mongodb/{{mongodb_instance}}/pid/config_mongod_{{item.port}}.pid
port={{item.port}}
bind_ip=0.0.0.0
replSet={{item.name}}
wiredTigerCacheSizeGB={{item.memory}}
configsvr=true
timeZoneInfo=/usr/share/zoneinfo
#mongos认证模板
vim mongos_mongodb_auth.conf.j2
logpath=/kingdee/mongodb/{{mongodb_instance}}/logs/mongos_{{item.port}}.log
logappend = true
fork=true
pidfilepath=/kingdee/mongodb/{{mongodb_instance}}/pid/mongos_{{item.port}}.pid
port={{item.port}}
bind_ip=0.0.0.0
timeZoneInfo=/usr/share/zoneinfo
{%for config in mongodb_config %}
configdb={{config.name}}/{{config.ip.split(",")[0]}}:{{config.port}},{{config.ip.split(",")[1]}}:{{config.port}},{{config.ip.split(",")[2]}}:{{config.port}}
{%endfor%}
keyFile=/kingdee/mongodb/{{mongodb_instance}}/bin/keyFile
#mongos无需认证模板
vim mongos_mongodb_unauth.conf.j2
logpath=/kingdee/mongodb/{{mongodb_instance}}/logs/mongos_{{item.port}}.log
fork=true
logappend=true
pidfilepath=/kingdee/mongodb/{{mongodb_instance}}/pid/mongos_{{item.port}}.pid
port={{item.port}}
bind_ip=0.0.0.0
timeZoneInfo=/usr/share/zoneinfo
{%for config in mongodb_config %}
configdb={{config.name}}/{{config.ip.split(",")[0]}}:{{config.port}},{{config.ip.split(",")[1]}}:{{config.port}},{{config.ip.split(",")[2]}}:{{config.port}}
{%endfor%}
#shard无需认证模板
vim shards_mongodb_unauth.conf.j2
logpath= /kingdee/mongodb/{{mongodb_instance}}/logs/mongod_{{item.port}}.log
dbpath=/kingdee/mongodb/{{mongodb_instance}}/data/{{item.name}}
shardsvr=true
fork=true
pidfilepath=/kingdee/mongodb/{{mongodb_instance}}/pid/mongod_{{item.port}}.pid
port={{item.port}}
bind_ip=0.0.0.0
replSet={{item.name}}
wiredTigerCacheSizeGB={{item.memory}}
timeZoneInfo=/usr/share/zoneinfo
#shard认证模板
vim shards_mongodb_auth.conf.j2
logpath= /kingdee/mongodb/{{mongodb_instance}}/logs/mongos_{{item.port}}.log
dbpath=/kingdee/mongodb/{{mongodb_instance}}/data/{{item.name}}
pidfilepath=/kingdee/mongodb/{{mongodb_instance}}/pid/mongos_{{item.port}}.pid
port={{item.port}}
bind_ip=0.0.0.0
replSet={{item.name}}
wiredTigerCacheSizeGB={{item.memory}}
keyFile={{mongodb_dir}}/{{mongodb_instance}}/bin/keyFile
timeZoneInfo=/usr/share/zoneinfo
shardsvr=true
fork=true
auth=true
#shards启停脚本
vim shards_mongodb.start.sh.j2
start(){
{{mongodb_dir}}/{{mongodb_instance}}/bin/mongod -f {{mongodb_dir}}/{{mongodb_instance}}/bin/{{item.name}}_{{item.port}}.conf
}
stop(){
{{mongodb_dir}}/{{mongodb_instance}}/bin/mongod -f {{mongodb_dir}}/{{mongodb_instance}}/bin/{{item.name}}_{{item.port}}.conf --shutdown
}
case "$1" in
start)
start
;;
stop)
stop
;;
restart)
stop
sleep 10
start
;;
*)
echo "Only param { start|stop|restart } is supported !"
esac
#mongos启动脚本
vim mongos_mongodb.start.sh.j2
start(){
{{mongodb_dir}}/{{mongodb_instance}}/bin/mongos -f {{mongodb_dir}}/{{mongodb_instance}}/bin/{{item.name}}_{{item.port}}.conf
}
stop(){
ps -ef|grep {{mongodb_dir}}/{{mongodb_instance}}/bin/mongos|grep -v grep|awk '{print "kill -s 9 "$2}'|sh
}
case "$1" in
start)
start
;;
stop)
stop
;;
restart)
stop
sleep 10
start
;;
*)
echo "Only param { start|stop|restart } is supported !"
esac
3.二进制文件准备:
cd roles/mongodb_single/files
#生产随机码
openssl rand -base64 50 > fileKey
#将文件copy到给目录下:
mongod, mongodump,mongoexport, mongofiles,mongoimport,mongorestore,mongos,mongosh,mongostat
4.编写playbook
cd roles/mongodb_shards/tasks
#部署shards
vim install_shards.yml
- name: "groupadd group"
group:
name: "{{mongodb_start_user}}"
gid: "9999"
state: "present"
when: ansible_ssh_host in item.ip.split(',')
with_items:
"{{mongodb_shards}}"
ignore_errors: yes
- name: "useradd {{mongodb_start_user}}"
user:
name: "{{mongodb_start_user}}"
uid: "9999"
gid: "mongodb_start_user"
when: ansible_ssh_host in item.ip.split(',')
with_items:
"{{mongodb_shards}}"
ignore_errors: yes
- name: "mkdir {{mongodb_dir}}"
file:
path: "{{mongodb_dir}}/{{mongodb_instance}}/{{item}}"
state: "directory"
owner: "{{mongodb_start_user}}"
group: "{{mongodb_start_user}}"
mode: 0755
recurse: yes
with_items:
- logs
- bin
- pid
- data
ignore_errors: yes
- name: "copy mongodb bin to {{mongodb_dir}}/{{mongodb_instance}}/bin"
copy:
src: "{{item}}"
dest: "{{mongodb_dir}}/{{mongodb_instance}}/bin/{{item}}"
owner: "{{mongodb_start_user}}"
group: "{{mongodb_start_user}}"
mode: 0755
with_items:
- mongod
- mongodump
- mongoexport
- mongofiles
- mongoimport
- mongorestore
- mongos
- mongosh
- mongostat
- mongotop
ignore_errors: yes
- name: "create file key to {{mongodb_dir}}/{{mongodb_instance}}/bin"
copy:
src: "fileKey"
dest: "{{mongodb_dir}}/{{mongodb_instance}}/bin/keyFile"
owner: "{{mongodb_start_user}}"
group: "{{mongodb_start_user}}"
mode: 0400
when: ansible_ssh_host in item.ip.split(',')
with_items:
"{{mongodb_shards}}"
- name: "mkdir mongodb_shard directroy"
file:
path: "{{mongodb_dir}}/{{mongodb_instance}}/data/{{item.name}}"
state: "directory"
owner: "{{mongodb_start_user}}"
group: "{{mongodb_start_user}}"
when: ansible_ssh_host in item.ip.split(',')
with_items:
"{{mongodb_shards}}"
- name: "copy mongodb_shard unauth config"
template:
src: shards_mongodb_unauth.conf.j2
dest: "{{mongodb_dir}}/{{mongodb_instance}}/bin/{{item.name}}_{{item.port}}_unauth.conf"
when: ansible_ssh_host in item.ip.split(',')
with_items:
"{{mongodb_shards}}"
- name: "copy mongodb_shard auth config"
template:
src: shards_mongodb_auth.conf.j2
dest: "{{mongodb_dir}}/{{mongodb_instance}}/bin/{{item.name}}_{{item.port}}.conf"
when: ansible_ssh_host in item.ip.split(',')
with_items:
"{{mongodb_shards}}"
- name: "copy mongodb_shard start file"
template:
src: "shards_mongodb.start.sh.j2"
dest: "{{mongodb_dir}}/{{mongodb_instance}}/bin/start_{{item.name}}_{{item.port}}.sh"
when: ansible_ssh_host in item.ip.split(',')
with_items:
"{{mongodb_shards}}"
- name: "start mongodb_shard start file"
shell: |
{{mongodb_dir}}/{{mongodb_instance}}/bin/mongod -f {{mongodb_dir}}/{{mongodb_instance}}/bin/{{item.name}}_{{item.port}}_unauth.conf
when: ansible_ssh_host in item.ip.split(',')
with_items:
"{{mongodb_shards}}"
#部署config
vim install_config.yml
- name: "groupadd group"
group:
name: "{{mongodb_start_user}}"
gid: "9999"
state: "present"
when: ansible_ssh_host in item.ip.split(',')
with_items:
"{{mongodb_shards}}"
ignore_errors: yes
- name: "useradd {{mongodb_start_user}}"
user:
name: "{{mongodb_start_user}}"
uid: "9999"
gid: "mongodb_start_user"
when: ansible_ssh_host in item.ip.split(',')
with_items:
"{{mongodb_shards}}"
ignore_errors: yes
- name: "mkdir {{mongodb_dir}}"
file:
path: "{{mongodb_dir}}/{{mongodb_instance}}/{{item}}"
state: "directory"
owner: "{{mongodb_start_user}}"
group: "{{mongodb_start_user}}"
mode: 0755
recurse: yes
with_items:
- logs
- bin
- pid
- data
ignore_errors: yes
- name: "copy mongodb bin to {{mongodb_dir}}/{{mongodb_instance}}/bin"
copy:
src: "{{item}}"
dest: "{{mongodb_dir}}/{{mongodb_instance}}/bin/{{item}}"
owner: "{{mongodb_start_user}}"
group: "{{mongodb_start_user}}"
mode: 0755
with_items:
- mongod
- mongodump
- mongoexport
- mongofiles
- mongoimport
- mongorestore
- mongos
- mongosh
- mongostat
- mongotop
ignore_errors: yes
- name: "create file key to {{mongodb_dir}}/{{mongodb_instance}}/bin"
copy:
src: "fileKey"
dest: "{{mongodb_dir}}/{{mongodb_instance}}/bin/keyFile"
owner: "{{mongodb_start_user}}"
group: "{{mongodb_start_user}}"
mode: 0400
when: ansible_ssh_host in item.ip.split(',')
with_items:
"{{mongodb_config}}"
- name: "mkdir mongodb_config directroy"
file:
path: "{{mongodb_dir}}/{{mongodb_instance}}/data/{{item.name}}"
state: "directory"
owner: "{{mongodb_start_user}}"
group: "{{mongodb_start_user}}"
when: ansible_ssh_host in item.ip.split(',')
with_items:
"{{mongodb_config}}"
- name: "copy mongodb_config unauth config"
template:
src: config_mongodb_unauth.conf.j2
dest: "{{mongodb_dir}}/{{mongodb_instance}}/bin/{{item.name}}_{{item.port}}_unauth.conf"
when: ansible_ssh_host in item.ip.split(',')
with_items:
"{{mongodb_config}}"
- name: "copy mongodb_config auth config"
template:
src: config_mongodb_auth.conf.j2
dest: "{{mongodb_dir}}/{{mongodb_instance}}/bin/{{item.name}}_{{item.port}}.conf"
when: ansible_ssh_host in item.ip.split(',')
with_items:
"{{mongodb_config}}"
- name: "copy mongodb_config start file"
template:
src: "shards_mongodb.start.sh.j2"
dest: "{{mongodb_dir}}/{{mongodb_instance}}/bin/start_{{item.name}}_{{item.port}}.sh"
when: ansible_ssh_host in item.ip.split(',')
with_items:
"{{mongodb_config}}"
- name: "start mongodb_config start file"
shell: |
{{mongodb_dir}}/{{mongodb_instance}}/bin/mongod -f {{mongodb_dir}}/{{mongodb_instance}}/bin/{{item.name}}_{{item.port}}_unauth.conf
when: ansible_ssh_host in item.ip.split(',')
with_items:
"{{mongodb_config}}"
#部署mongos
vim install_mongos.yml
- name: "groupadd group"
group:
name: "{{mongodb_start_user}}"
gid: "9999"
state: "present"
when: ansible_ssh_host in item.ip.split(',')
with_items:
"{{mongodb_shards}}"
ignore_errors: yes
- name: "useradd {{mongodb_start_user}}"
user:
name: "{{mongodb_start_user}}"
uid: "9999"
gid: "mongodb_start_user"
when: ansible_ssh_host in item.ip.split(',')
with_items:
"{{mongodb_shards}}"
ignore_errors: yes
- name: "mkdir {{mongodb_dir}}"
file:
path: "{{mongodb_dir}}/{{mongodb_instance}}/{{item}}"
state: "directory"
owner: "{{mongodb_start_user}}"
group: "{{mongodb_start_user}}"
mode: 0755
recurse: yes
with_items:
- logs
- bin
- pid
- data
ignore_errors: yes
- name: "copy mongodb bin to {{mongodb_dir}}/{{mongodb_instance}}/bin"
copy:
src: "{{item}}"
dest: "{{mongodb_dir}}/{{mongodb_instance}}/bin/{{item}}"
owner: "{{mongodb_start_user}}"
group: "{{mongodb_start_user}}"
mode: 0755
with_items:
- mongod
- mongodump
- mongoexport
- mongofiles
- mongoimport
- mongorestore
- mongos
- mongosh
- mongostat
- mongotop
ignore_errors: yes
- name: "create file key to {{mongodb_dir}}/{{mongodb_instance}}/bin"
copy:
src: "fileKey"
dest: "{{mongodb_dir}}/{{mongodb_instance}}/bin/keyFile"
owner: "{{mongodb_start_user}}"
group: "{{mongodb_start_user}}"
mode: 0400
when: ansible_ssh_host in item.ip.split(',')
with_items:
"{{mongodb_mongos}}"
- name: "mkdir mongodb_mongos directroy"
file:
path: "{{mongodb_dir}}/{{mongodb_instance}}/data/{{item.name}}"
state: "directory"
owner: "{{mongodb_start_user}}"
group: "{{mongodb_start_user}}"
when: ansible_ssh_host in item.ip.split(',')
with_items:
"{{mongodb_mongos}}"
- name: "copy mongodb_mongos unauth config"
template:
src: mongos_mongodb_unauth.conf.j2
dest: "{{mongodb_dir}}/{{mongodb_instance}}/bin/{{item.name}}_{{item.port}}_unauth.conf"
when: ansible_ssh_host in item.ip.split(',')
with_items:
"{{mongodb_mongos}}"
- name: "copy mongodb_mongos auth config"
template:
src: mongos_mongodb_auth.conf.j2
dest: "{{mongodb_dir}}/{{mongodb_instance}}/bin/{{item.name}}_{{item.port}}.conf"
when: ansible_ssh_host in item.ip.split(',')
with_items:
"{{mongodb_mongos}}"
- name: "copy mongodb_mongos start file"
template:
src: "mongos_mongodb.start.sh.j2"
dest: "{{mongodb_dir}}/{{mongodb_instance}}/bin/start_{{item.name}}_{{item.port}}.sh"
when: ansible_ssh_host in item.ip.split(',')
with_items:
"{{mongodb_mongos}}"
- name: "start mongodb_mongos start file"
shell: |
{{mongodb_dir}}/{{mongodb_instance}}/bin/mongos -f {{mongodb_dir}}/{{mongodb_instance}}/bin/{{item.name}}_{{item.port}}_unauth.conf
when: ansible_ssh_host in item.ip.split(',')
with_items:
"{{mongodb_mongos}}"
#初始化集群
vim initiate_shards.yml
---
- name: "initiate mongodb shards cluster"
shell: |
{{mongodb_dir}}/{{mongodb_instance}}/bin/mongosh --port {{item.port}} --host {{item.ip.split(',')[0]}} admin --eval 'rs.initiate({_id: "{{item.name}}",members: [{ _id: 0,host: "{{item.ip.split(",")[0]}}:{{item.port}}"},{ _id: 1,host: "{{item.ip.split(",")[1]}}:{{item.port}}"},{ _id: 2,host: "{{item.ip.split(",")[2]}}:{{item.port}}"}]})'
when: ansible_ssh_host == item.ip.split(',')[0]
with_items:
"{{mongodb_shards}}"
ignore_errors: yes
- name: "sleep 20s for wating initate !!"
shell: sleep 20
- name: "initiate mongodb config cluster"
shell: |
{{mongodb_dir}}/{{mongodb_instance}}/bin/mongosh --port {{item.port}} --host {{item.ip.split(',')[0]}} admin --eval 'rs.initiate({ _id: "{{item.name}}",configsvr: true,members: [{ _id: 0,host: "{{item.ip.split(",")[0]}}:{{item.port}}"},{ _id: 1,host: "{{item.ip.split(",")[1]}}:{{item.port}}"},{ _id: 2,host: "{{item.ip.split(",")[2]}}:{{item.port}}"}]})'
when: ansible_ssh_host == item.ip.split(',')[0]
with_items:
"{{mongodb_config}}"
#添加分片
vim addShard.yml
---
- name: "addshrds mongodb"
shell: |
{{mongodb_dir}}/{{mongodb_instance}}/bin/mongosh --port 20000 --host {{item.ip.split(',')[0]}} admin --eval 'sh.addShard("{{item.name}}/{{item.ip.split(",")[0]}}:{{item.port}},{{item.ip.split(",")[1]}}:{{item.port}},{{item.ip.split(",")[2]}}:{{item.port}}")'
when: ansible_ssh_host == item.ip.split(',')[0]
with_items:
"{{mongodb_shards}}"
#创建用户及开启mongodb认证
vim createUser.yml
---
- name: "create mongodbuser for mongodb"
shell: |
{{mongodb_dir}}/{{mongodb_instance}}/bin/mongosh --port {{item.port}} --host {{ansible_ssh_host}} admin --eval 'db.createUser({user:"{{mongodbuser}}",pwd:"{{mongodbpass}}",roles:[{role:"root",db:"admin"}]})'
when: ansible_ssh_host == item.ip.split(',')[0]
with_items:
"{{mongodb_mongos}}"
ignore_errors: yes
######remove unauth config#####################
- name: "remove shard unauth.config"
file:
path: "{{mongodb_dir}}/{{mongodb_instance}}/bin/{{item.name}}_{{item.port}}_unauth.conf"
state: absent
when: ansible_ssh_host in item.ip.split(',')
with_items:
"{{mongodb_shards}}"
- name: "remove config unauth.config"
file:
path: "{{mongodb_dir}}/{{mongodb_instance}}/bin/{{item.name}}_{{item.port}}_unauth.conf"
state: absent
when: ansible_ssh_host in item.ip.split(',')
with_items:
"{{mongodb_config}}"
- name: "remove mongos unauth.config"
file:
path: "{{mongodb_dir}}/{{mongodb_instance}}/bin/{{item.name}}_{{item.port}}_unauth.conf"
state: absent
when: ansible_ssh_host in item.ip.split(',')
with_items:
"{{mongodb_mongos}}"
######restart mongodb#####################
- name: "restart config mongodb"
shell: |
sh {{mongodb_dir}}/{{mongodb_instance}}/bin/start_{{item.name}}_{{item.port}}.sh restart
when: ansible_ssh_host in item.ip.split(',')
with_items:
"{{mongodb_config}}"
- name: "restart shard mongodb "
shell: |
sh {{mongodb_dir}}/{{mongodb_instance}}/bin/start_{{item.name}}_{{item.port}}.sh restart
when: ansible_ssh_host in item.ip.split(',')
with_items:
"{{mongodb_shards}}"
- name: "restart mongos mongodb"
shell: |
sh {{mongodb_dir}}/{{mongodb_instance}}/bin/start_{{item.name}}_{{item.port}}.sh restart
when: ansible_ssh_host in item.ip.split(',')
with_items:
"{{mongodb_mongos}}"
vim main.yml
---
- include_tasks: install_shards.yml
- include_tasks: install_config.yml
- include_tasks: initiate_shards.yml
- include_tasks: install_mongos.yml
- include_tasks: addShard.yml
- include_tasks: createUser.yml
tasks下文件如下:
四.编写hosts文件,与roles文件在同级目录:
vim hosts
[mongodb_shards]
mongodb-shard1 ansible_ssh_host=192.168.126.128
mongodb-shard2 ansible_ssh_host=192.168.126.129
mongodb-shard3 ansible_ssh_host=192.168.126.130
五.编写运行yml文件:
vim startinstall_shareds_mongodb.yml
---
- hosts: mongodb
remote_user: yzj
#become_method: sudo
#$become_user: root
#$become: yes
gather_facts: no
vars:
- mongodb_dir: /kingdee/mongodb
- mongodb_start_user: yzj
- mongodbuser: yzjuser
- mongodbpass: Mytest@@@@@1203
- mongodb_mongos:
- {"name": "mongos","ip":"192.168.126.128,192.168.126.129,192.168.126.130","port":"20000","memory":"1"}
- mongodb_config:
- {"name": "configs","ip":"192.168.126.128,192.168.126.129,192.168.126.130","port":"20001","memory":"2"}
- mongodb_shards:
- {"name": "shard1","ip": "192.168.126.128,192.168.126.129,192.168.126.130","port":"27017","memory":"2"}
- {"name": "shard2","ip": "192.168.126.128,192.168.126.129,192.168.126.130","port":"27018","memory":"2"}
- {"name": "shard3","ip": "192.168.126.128,192.168.126.129,192.168.126.130","port":"27019","memory":"2"}
- mongodb_instance: datacore
roles:
- role: mongodb_shards
ansible-playbook目录如下:
六,执行部署三分片集群:
ansible-playbook -i hosts startinstall_shards_mongodb.yml
七,检查集群状态
/kingdee/mongodb/datacore/bin/mongosh --port 20000 admin
[direct: mongos] admin> show dbs
MongoServerError: command listDatabases requires authentication
[direct: mongos] admin> db.auth("yzjuser","Mytest@@@@@1203")
#检查分片:
db.runCommand({listshards:1})
#查看分片状态:
db.printShardingStatus()
至此,ansible-playbook部署mongodb分片已完成,下面会讲到elasticsearch[单机、集群]使用ansible-playbook进行部署