#Word文档导入#Ceph分布式存储搭建

主机信息规划​

环境配置​

1.1主机名修改安装计划的修改​

hostnamectl set-hostname ceph205.cn​

hostnamectl set-hostname ceph206.cn​

hostnamectl set-hostname ceph207.cn​

1.2 修改主机映射​

vi /etc/hosts​

192.168.100.205 ceph205​

192.168.100.206 ceph206​

192.168.100.207 ceph207​

1.3 免密钥认证​

[root@ceph205 ~]# ssh-keygen #所有的输入选项都直接回车生成。​

[root@ceph205 ~]# ssh-copy-id ceph205 #然后yes 输入各主机的密码​

[root@ceph205 ~]# ssh-copy-id ceph206​

[root@ceph205 ~]# ssh-copy-id ceph207​

1.4修改系统源​

yum install -y wget #先下载wget工具​

[root@ceph205 ~]# mkdir /etc/yum.repos.d/yum/​

[root@ceph205 ~]# mv /etc/yum.repos.d/*.repo /etc/yum.repos.d/yum/ #备份源​

[root@ceph205~]#wget -O /etc/yum.repos.d/CentOS-Base.repo http://mirrors.aliyun.com/repo/Centos-7.repo​

[root@ceph205 ~]# wget -O /etc/yum.repos.d/epel.repo http://mirrors.aliyun.com/repo/epel-7.repo​

[root@ceph205 ~]# vi /etc/yum.repos.d/ceph.repo #编辑ceph源​

[ceph]​

name=ceph​

baseurl=http://mirrors.aliyun.com/ceph/rpm-jewel/el7/x86_64/​

gpgcheck=0​

priority=1​

[ceph-noarch]​

name=cephnoarch​

baseurl=http://mirrors.aliyun.com/ceph/rpm-jewel/el7/noarch/​

gpgcheck=0​

priority=1​

[ceph-source]​

name=Ceph source packages​

baseurl=http://mirrors.aliyun.com/ceph/rpm-jewel/el7/SRPMS​

gpgcheck=0​

priority=1​

yum clean all ; yum makecache #刷新源​

1.5时间同步​

rm /etc/localtime #删除link​

ln -vs /usr/share/zoneinfo/Asia/Shanghai /etc/localtime #软链接link​

yum install ntpdate -y​

ntpdate time.nist.gov​

1.6关闭防火墙防止影响操作​

systemctl stop firewalld ;systemctl disable firewalld ; iptables -F ;setenforce 0​

1.6Ceph205操作​

拷贝主机映射到其他节点​

scp /etc/hosts ceph206:/etc/hosts​

scp /etc/hosts ceph207:/etc/hosts​

分区格式​

2.1格式化第二块盘并挂载开机启动​

[root@ceph205 ~]# mkfs.xfs /dev/sdb1 -f​

[root@ceph205 ~]# mkdir /var/data1​

[root@ceph205 ~]# mount /dev/sdb1 /var/data1​

[root@ceph205 ~] # echo "/dev/sdb1 /var/data1 xfs defaults 0 0" >> /etc/fstab #开机挂载​

mount -a​

[root@ceph206 ~]# mkfs.xfs /dev/sdb1 -f​

[root@ceph206 ~]# mkdir /var/data2​

[root@ceph206 ~]# mount /dev/sdb1 /var/data2​

[root@ceph206 ~]# echo "/dev/sdb1 /var/data2 xfs defaults 0 0" >> /etc/fstab​

mount -a​

[root@ceph207 ~]# mkfs.xfs /dev/sdb1 -f​

[root@ceph207 ~]# mkdir /var/data3.1​

[root@ceph207 ~]# mount /dev/sdb1 /var/data3.1​

[root@ceph207 ~]# echo "/dev/sdb1 /var/data3.1 xfs defaults 0 0" >> /etc/fstab​

mount -a​

安装ceph-deploy 管理工具​

3.1修改ceph配置文件​

[root@ceph205 ~]# yum -y install ceph-deploy​

创建monitor服务​

[root@ceph205 ~]# mkdir /etc/ceph && cd /etc/ceph​

[root@ceph205 ceph]# ceph-deploy new ceph205 #安装控制节点到ceph205​

[root@ceph205 ~]# vi /etc/ceph/ceph.conf #设置服务器默认存储两份也可以正常读写​

osd_pool_default_size = 2​

public_network= 192.168.100.0/24 # 为多mon做准备​

3.2各节点安装软件​

[root@ceph205 ~]# ceph-deploy install ceph205 ceph206 ceph207​

安装ceph monitor​

cd /etc/ceph​

[root@ceph205 ceph]# ceph-deploy mon create ceph205​

[root@ceph205 ceph]# ceph-deploy gatherkeys ceph205 # 收集节点的keyring文件​

激活osd​

[root@ceph205 ceph]# ceph-deploy osd prepare ceph205:/var/data1 ceph206:/var/data2 ceph207:/var/data3.1 # 准备使用的硬盘​

各个目录给权限​

chmod 777 -R /var/data1​

chmod 777 -R /var/data2​

chmod 777 -R /var/data3.1​

复制管理秘钥到各节点​

cd /etc/ceph/​

ceph-deploy admin ceph205 ceph206 ceph207​

各个节点操作给认证读取权限​

chmod +r /etc/ceph/ceph.client.admin.keyring​

激活osd 服务​

ceph-deploy osd activate ceph205:/var/data1 ceph206:/var/data2 ceph207:/var/data3.1​

部署mds服务​

[root@ceph205 ceph]# ceph-deploy mds create ceph206 ceph207 #我们MDS安装2台​

[root@ceph205 ceph]# ceph mds stat #查看状态​

查看集群信息​

ceph -s​

rados df#查看使用率​

3.3添加多mon配置 ​

参考http://docs.ceph.org.cn/start/quick-ceph-deploy/#monitors​

vi /etc/ceph/ceph.conf​

public_network= 192.168.100.0/24 #前面没有添加的话加上​

ceph-deploy --overwrite-conf config push ceph205 ceph206 ceph207 #同步配置​

ceph-deploy admin ceph205 ceph206 ceph207 #同步配置​

ceph-deploy mon add ceph206 #添加mon服务​

ceph-deploy mon add ceph207#添加mon服务​

ceph quorum_status --format json-pretty #查看投票者mon大于一半的存活系统就可以正常了​

ceph mon stat #查看状态​

挂载存储准备​

4.1 创建存储数据用的数据 和数据元池​

ceph osd pool create cephdata 128​

ceph osd pool create cephdata2 128​

ceph fs new 128 cephdata cephdata2​

ceph fs ls #查看文件系统​

4.2 git208.cn上挂载​

4.2.1环境配置​

hostnamectl set-hostname git208.cn #设置主机名​

yum install -y wget​

mkdir /etc/yum.repos.d/yum/​

mv /etc/yum.repos.d/*.repo /etc/yum.repos.d/yum/​

wget -O /etc/yum.repos.d/CentOS-Base.repo http://mirrors.aliyun.com/repo/Centos-7.repo​

wget -O /etc/yum.repos.d/epel.repo http://mirrors.aliyun.com/repo/epel-7.repo​

yum clean all ; yum makecache​

时间同步​

rm /etc/localtime #删除link​

ln -vs /usr/share/zoneinfo/Asia/Shanghai /etc/localtime #软件link​

yum install ntpdate -y​

ntpdate time.nist.gov​

关闭防火墙​

systemctl stop firewalld ;systemctl disable firewalld ; iptables -F ;setenforce 0​

vi /etc/yum.repos.d/ceph.repo 配置ceph源​

[ceph]​

name=ceph​

baseurl=http://mirrors.aliyun.com/ceph/rpm-jewel/el7/x86_64/​

gpgcheck=0​

priority=1​

[ceph-noarch]​

name=cephnoarch​

baseurl=http://mirrors.aliyun.com/ceph/rpm-jewel/el7/noarch/​

gpgcheck=0​

priority=1​

[ceph-source]​

name=Ceph source packages​

baseurl=http://mirrors.aliyun.com/ceph/rpm-jewel/el7/SRPMS​

gpgcheck=0​

priority=1​

yum clean all ; yum makecache #刷新源缓存​

4.2.2 安装ceph到客户端​

ssh-copy-id git208 #免密钥认证​

ceph-deploy install git208 #安装到客户端​

scp /etc/ceph/* git208:/etc/ceph/ #205上执行复制认证文件到客户端端口​

chmod +r /etc/ceph/ceph.client.admin.keyring #在客户端上执行给认证文件读取权限​

内核驱动挂载Ceph文件系统​

[root@git208 ~]# mkdir /gitlab-data #创建挂载点​

[root@git208 ~]# cat /etc/ceph/ceph.client.admin.keyring​

[client.admin]​

key = AQDNBQFca7UYGxAA6wOfoZR4JWdP5yM56S8DeQ==​

# 将key对应的值复制下来保存到文件：/etc/ceph/admin.secret中。​

[root@xuegod65 ~]# vi /etc/ceph/admin.secret​

AQDNBQFca7UYGxAA6wOfoZR4JWdP5yM56S8DeQ==​

测试挂载​

[root@git208 ~]# mount -t ceph 192.168.100.205:6789:/ /gitlab-data -o name=admin,secretfile=/etc/ceph/admin.secret #指定验证文件挂载​

[root@git208 ~]# umount /gitlab-data #先卸载​

开机挂载多路径​

[root@git208 ~]# vi /etc/fstab​

192.168.100.205,192.168.100.206,192.168.100.207:6789:/ /gitlab-data ceph name=admin,secretfile=/etc/ceph/admin.secret,_netdev​

[root@git208 ~]# mount -a #挂载​

[root@git208 ~]# chmod -R 777 /gitlab-data #目录先给所有读写权限 后期再根据情况调试​

配置docker环境在git208.cn上​

5.1 配置环境​

yum install -y yum-utils device-mapper-persistent-data lvm2​

yum-config-manager --add-repo ​​http://mirrors.aliyun.com/docker-ce/linux/centos/docker-ce.repo​​ #导入yum源​

yum clean all ; yum makecache #刷新yum源​

yum install docker-ce -y #安装docker ​

#开启docker服务 并设置开机自动启动​

systemctl start docker && systemctl enable docker.service && systemctl status docker dcker​

配置阿里云加速加速docker拉取速度​

cat >> /etc/docker/daemon.json <<EOF​

{ "registry-mirrors": ["https://rncxm540.mirror.aliyuncs.com"] }​

EOF​

systemctl daemon-reload && systemctl restart docker #加载配置生效​

docker pull docker.io/twang2218/gitlab-ce-zh #拉取gitlabdocker​

报错​

Get https://registry-1.docker.io/v2/: x509: certificate has expired or is not yet valid​

date -s "Fri May 31 11:16:31 CST 2019" #时间不对修改时间后正常​

5.2 启动gitlab 的docker​

mkdir /srv/gitlab #创建放docker映射的文件目录​

在git208.cn上启动docker​

sudo docker run --detach \​

-u root \​

--hostname 192.168.100.208 \​

--env GITLAB_OMNIBUS_CONFIG="external_url 'http://192.168.100.208/'; gitlab_rails['lfs_enabled'] = true;" \​

--publish 443:443 --publish 80:80 --publish 2222:22 \​

--name gitlab \​

--restart always \​

--volume /srv/gitlab/config:/etc/gitlab \​

--volume /srv/gitlab/logs:/var/log/gitlab \​

--volume /srv/gitlab/data:/var/opt/gitlab \​

docker.io/twang2218/gitlab-ce-zh​

正常启动的状态​

服务正常后登陆web后台设置密码为gitlab443!​

​​http://192.168.100.208/​​​

6.Ceph集群管理常用命令​

ceph health​

ceph df #空间使用情况​

ceph osd lost 1 #1丢失​

ps -ef | grep ceph #查看服务运行​

ceph health detail​

ceph osd tree #查询osd信息id号​

ceph -w #监控集群信息变动​

ceph 单节点故障处理​

ceph health detail​

ceph osd out osd_id​

ceph osd crush remove osd_name​

ceph auth del osd_name​

ceph osd rm osd_id​

操作参考2​

ceph osd reweight 2 0.0 #调节移除节点2的权重为0​

ceph osd out 2 移除集群的中的2节点​

ceph osd crush remove osd.2​

ceph auth del osd.2​

ceph osd rm 2​

7.虚拟机数据同步测试1分钟更新一次记录时间​

测试​

sudo docker run --detach \​

-u root \​

--hostname 192.168.110.45 \​

--env GITLAB_OMNIBUS_CONFIG="external_url 'http://192.168.110.45/'; gitlab_rails['lfs_enabled'] = true;" \​

--publish 443:443 --publish 80:80 --publish 2222:22 \​

--name gitlab \​

--restart always \​

--volume /srv/gitlab/config:/etc/gitlab \​

--volume /srv/gitlab/logs:/var/log/gitlab \​

--volume /srv/gitlab/data:/var/opt/gitlab \​

docker.io/twang2218/gitlab-ce-zh​

http://192.168.110.45​

root​

gitlab443!​

u​

群组test项目 xiangmu​

cd /opt2/​

git clone http://192.168.110.45/test/xiangmu.git​

root​

gitlab443!​

cd xiangmu/​

inux系统下添加测试文件​

278 vi index.html​

279 git add .​

281 git config --global user.name "root"​

283 git config --global user.email "root@245305579.com"​

284 git commit -m "123"​

286 git config --global push.default simple​

287 git push​

git pull​

vi /opt2/xiangmu/.git/config #记住密码配置​

[credential]​

helper = store​

vi /root/gx.sh​

chmod +x /root/gx.sh​

#/bin/bash​

#/bin/bash​

cd /opt2/xiangmu​

git pull > a.txt​

date >> b.txt​

crontab -e​

*/1 * * * * sh /root/gx.sh​

systemctl reload crond​

Opt2目录为挂载的存储目录​