k8s搭建ceph集群

1、准备三台机器：

172.19.230.222

172.19.230.224

172.19.230.226

2、关闭防火墙关闭selinux

systemctl stop firewalld
systemctl disable firewalld
setenforce 0
sed -i 's/^SELINUX=.*/SELINUX=disabled/' /etc/selinux/config

3、设置机器hostname

hostnamectl set-hostname ceph222
hostnamectl set-hostname ceph224
hostnamectl set-hostname ceph226

vi /etc/hosts
172.19.230.222 ceph222
172.19.230.224 ceph224
172.19.230.226 ceph226

4、使用VMware挂载未使用硬盘

5、机器时间同步

yum install ntpdate
timedatectl set-timezone Asia/Shanghai
ntpdate ntp3.aliyun.com
date

6、在三台机器分别安装配置docker

centos7 安装docker

7、安装Python

yum install -y python3 python3-devel

---

以下步骤仅在ceph222上执行即可

8、安装dnf命令

yum install epel-release -y
yum install dnf -y

9、创建ceph.repo

tee /etc/yum.repos.d/ceph.repo <<-'EOF'
[Ceph-noarch]
name=Ceph noarch
baseurl=http://mirrors.163.com/ceph/rpm-octopus/el7/noarch/
enabled=1
gpgcheck=0

[Ceph-source]
name=Ceph SRPMS
baseurl=http://mirrors.163.com/ceph/rpm-octopus/el7/x86_64/
enabled=1
gpgcheck=0
EOF

10、安装软件包

yum clean all && yum makecache #如果报错执行
dnf install cephadm  -y
yum info cephadm
#安装源
cephadm install
#开始初始化一个最小的集群
cephadm bootstrap --mon-ip 172.19.230.222
cephadm version

URL: https://ceph222:8443/

User: admin

Password: cm25sple5g

11、拷贝公钥其他节点

ssh-copy-id -f -i /etc/ceph/ceph.pub root@172.19.230.224
ssh-copy-id -f -i /etc/ceph/ceph.pub root@172.19.230.226

---------其他节点安装cephadm-------

12、其他节点安装cephadm

另外两台机器执行上面8、9、10三步

yum clean all && yum makecache #如果报错执行
dnf install cephadm  -y
yum info cephadm
#安装源
cephadm install
cephadm version

---------返回安装节点执行-------

13、添加节点打标签

ceph orch host add ceph222 172.19.230.222
ceph orch host add ceph224 172.19.230.224
ceph orch host add ceph226 172.19.230.226

 ceph orch host label add ceph222 _admin
 ceph orch host label add ceph224 _admin
 ceph orch host label add ceph226 _admin
 
 ceph orch host ls

14 调整配置

ceph orch apply mon 3

ceph orch ls

ceph -s

15、自动添加所有osd级初始化磁盘比以前的版本方便多了，

ceph orch apply osd --all-available-devices

16 部署nfs服务

#创建一个fs服务创建在222，224，226的服务器上
ceph fs volume create cephfs-nfs "ceph222,ceph224,ceph226"
#一主两从
ceph orch apply mds cephfs-nfs "ceph222,ceph224,ceph226"
#创建一个池
ceph osd pool create nfs-pools
#创建一个高可用nfs分别运行在ceph26,ceph27上
ceph orch apply nfs my-nfs nfs-pools --placement="ceph222,ceph224,ceph226"

安装docker

yum install -y yum-utils device-mapper-persistent-data lvm2
yum-config-manager --add-repo http://download.docker.com/linux/centos/docker-ce.repo
yum-config-manager --add-repo http://mirrors.aliyun.com/docker-ce/linux/centos/docker-ce.repo
yum -y install docker

systemctl start docker
systemctl enable docker  

 

dnf install cephadm -y

cephadm install


cephadm install ceph-common

hostnamectl set-hostname ceph1
hostnamectl set-hostname ceph2
hostnamectl set-hostname ceph3
vi /etc/hosts
172.19.230.211 ceph1
172.19.230.212 ceph2
172.19.230.213 ceph3

ceph orch host add ceph1 172.19.230.211
ceph orch host add ceph2 172.19.230.212
ceph orch host add ceph3 172.19.230.213


 ceph orch host label add ceph1 _admin
 ceph orch host label add ceph2 _admin
 ceph orch host label add ceph3 _admin

 ceph orch host ls


 ceph orch apply osd --all-available-devices

ceph-deploy admin ceph1 ceph2 ceph3


ceph fs volume create cephfs-demo "ceph1,ceph2"


ceph orch apply mds cephfs-demo "ceph1,ceph2,ceph3"

ceph orch apply nfs my-nfs nfs-pools --placement="ceph2,ceph3"

ceph orch apply nfs clickhouse-nfs nfs-pools --placement="ceph1"



scp /etc/ceph/ceph.client.admin.keyring root@172.19.230.211:/etc/ceph/
scp /etc/ceph/ceph.conf  root@172.19.230.211:/etc/ceph/


mount -t nfs 172.19.230.222:/nfs /mnt
nfs
mount -t ceph 172.19.230.211:6789,172.19.230.212:6789,172.19.230.213:6789:/ /mnt/ -o name=admin,secret=AQBTYYBjzfJdJxAAVFmzX16Bt9m+L+MDq9zL1A==

参考链接

https://blog.csdn.net/weixin_42562106/article/details/123536586

https://blog.whsir.com/post-6697.html