Ceph集群部署

• 先检查是否已经安装Ceph，如有请卸载

	//检查Ceph
	ceph -s
	//卸载Ceph(如果没有相关命令，请先安装ceph-deploy脚本)
	ceph-deploy forgetkeys
    ceph-deploy purge node1 node2 node3
    ceph-deploy purgedata node1 node2 node3

• 安装前准备

	//关闭防火墙 
	systemctl stop firewalld 
	systemctl disable firewalld 
	//查看防火墙状态 
	systemctl status firewalld 
	//临时关闭SElinux 
	setenforce 0
	//永久关闭SElinux
	vi /etc/selinux/config
	SELINUX=disabled

• 配置各节点间的/etc/hosts文件

cat << "EOT" > /etc/hosts
172.31.236.221 opennebula00
172.31.236.222 opennebula01
172.31.236.223 opennebula02
172.31.236.224 opennebula03
172.31.236.225 opennebula04
EOT

• 各节点安装NTP服务

//安装NTP服务
yum install ntp ntpdate ntp-doc
//校对系统时钟 
ntpdate 0.cn.pool.ntp.org

• 安装部署ssh免密登陆

ssh-keygen
ssh-copy-id root@node1
ssh-copy-id root@node2
ssh-copy-id root@node3

• 在控制节点上安装部署ceph

//安装相关依赖
yum install snappy leveldb gdisk python-argparse gperftools-libs -y 

• 修改ceph安装源

//添加yum 源,导入key,这里我使用最新的mimic版本
rpm --import 'https://download.ceph.com/keys/release.asc'
su -c 'rpm -Uvh https://download.ceph.com/rpm-mimic/el7/noarch/ceph-release-1-0.el7.noarch.rpm' 
vi /etc/yum.repos.d/ceph.repo
mimic版本源
[Ceph]
name=Ceph packages for $basearch
baseurl=https://mirrors.aliyun.com/ceph/rpm-mimic/el7/$basearch
enabled=1
gpgcheck=1
type=rpm-md
gpgkey=https://download.ceph.com/keys/release.asc
[Ceph-noarch]
name=Ceph noarch packages
baseurl=https://mirrors.aliyun.com/ceph/rpm-mimic/el7/noarch
enabled=1
gpgcheck=1
type=rpm-md
gpgkey=https://download.ceph.com/keys/release.asc
[ceph-source]
name=Ceph source packages
baseurl=https://mirrors.aliyun.com/ceph/rpm-mimic/el7/SRPMS
enabled=1
gpgcheck=1
type=rpm-md
gpgkey=https://download.ceph.com/keys/release.asc

• 在所有节点安装ceph

yum install ceph -y

• 检查包是否完整

rpm -qa |egrep -i "ceph|rados|rbd"

• 部署ceph集群

//控制节点安装ceph部署工具
yum install -y ceph-deploy
//创建配置文件目录并生成配置文件
mkdir my-cluster
cd my-cluster/
ceph-deploy new opennebula00
ceph-deploy new opennebula00 opennebula01 opennebula02 opennebula03 opennebula04 opennebula10 opennebula11 opennebula12 opennebula13 opennebula14
//安装ceph
ceph-deploy install opennebula00 opennebula01

• 初始化monitor

//初始monitor并收集所有密钥
 ceph-deploy mon create-initial	
//各节点部署mgr
 ceph-deploy mgr create opennebula00 opennebula01
// 复制key
 ceph-deploy admin opennebula00 opennebula01

• 手动增加mon

// 在正常的mon上导出monmap,好的节点写再前面，后面把所有的坏的节点加上即可
monmaptool --create --clobber --fsid xxx --add opennebula00 172.31.236.221:6789 --add control1 172.16.12.41:6789 --add control2 172.16.12.42:6789 /tmp/monmap
// 在坏的节点上把/var/lib/ceph/mon目录删掉
rm -rf /var/lib/ceph/mon
// 将keyring 文件传到其他坏的节点上
scp /var/lib/ceph/mon/ceph-opennebula00/keyring root@opennebula01:/tmp/
// 重做坏节点的mon
ceph-mon --cluster ceph -i opennebula01 --mkfs --monmap /tmp/monmap --keyring /tmp/keyring -c /etc/ceph/ceph.conf
> 做完后如果mon无法启动，请将/var/lib/ceph/mon/ceph-opennebula00下的store.db进行替换，如还有问题，请根据日志进行对应文件的替换与修改
// 最后加入mon监控集群
ceph mon add opennebula01 172.31.236.222
// 查看验证
ceph mon stat

• 添加OSD

ceph-deploy disk list opennebula00 opennebula01
 //添加OSD
 ceph-deploy osd create --data /dev/sda3 opennebula00
 ceph-deploy osd create --data /dev/sda3 opennebula01

如果无法挂载分区时，请先umount 分区，再重新添加OSD

• 开启Dashboard

//开启Dashboard
ceph mgr module enable dashboard
 //生成SSL证书
ceph dashboard create-self-signed-cert
//创建管理员及登录密码
 ceph dashboard set-login-credentials admin admin
 //确认验证
 ceph mgr services

• 创建存储池

systemctl restart ceph-mon@mon-node3
//创建存储池
ceph osd pool create one 128
//列出存储池
ceph osd lspools
//创建访问用户池的用户
ceph auth get-or-create client.libvirt mon 'profile rbd' osd 'profile rbd pool=one'
//获取用户密钥并分发到节点
ceph auth get-key client.libvirt | tee client.libvirt.key
ceph auth get client.libvirt -o ceph.client.libvirt.keyring
//修改配置文件
vi ceph.conf

[global]
rbd_default_format = 2

//复制密钥环到各个节点
scp ceph.client.libvirt.keyring root@opennebula00:/etc/ceph
scp client.libvirt.key oneadmin@opennebula00:

• 节点设置

UUID=`uuidgen`; echo $UUID

cat > secret.xml <<EOF
<secret ephemeral='no' private='no'>
 <uuid>$UUID</uuid>
 <usage type='ceph'>
 <name>client.libvirt secret</name>
 </usage>
</secret>
EOF

//拷贝密钥到各节点
scp secret.xml oneadmin@opennebula00:

//为各节点上的libvirt用户定义密钥，并移除key
virsh -c qemu:///system secret-define secret.xml
virsh -c qemu:///system secret-set-value --secret $UUID --base64 $(cat client.libvirt.key)
rm client.libvirt.key
//验证配置
ssh oneadmin@opennebula00
rbd ls -p one --id libvirt