docker 安装 ceph 集群

• 机器

10.10.10.10    master
10.10.20.20    slave
10.10.10.30    slave

• 条件

1. 均安装了 docker
2. ceph 数据盘挂载在 /home/service/var
3. 目录存在 /var/lib/docker/

• 设置 master 通过 ssh 免密登陆 slaves

ssh-keygen
modify slave /etc/hosts.allow
master .ssh/id_rsa.pub >> slave .ssh/authorized_keys
ssh root@10.10.20.20
ssh root@10.10.30.30

• 安装

# 在 master 机器上
cd /root/my_ceph_cluster/
bash setup.sh
bash build.sh

• 测试

# master
cat /var/lib/docker/ceph_data/etc/ceph.client.admin.keyring

# 测试机器上
mkdir /mnt/cephfs
mount -t ceph 10.10.10.10:6789,10.10.20.20:6789,10.10.30.30:6789:/ /mnt/cephfs/ -o name=admin,secret=xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx=

dd if=/dev/zero of=test_zero bs=1M count=1024
dd if=/dev/urandom of=test_random bs=1M count=1024

脚本

• /root/my_ceph_cluster/setup.sh

#!/bin/bash

cat << EOFconf > cluster_conf.sh
cmd_master=10.10.10.10
cmd_slave_a=10.10.20.20
cmd_slave_b=10.10.30.30
ceph_public_network=10.10.10.0/24,10.10.20.0/24,10.10.30.0/24
ceph_daemon_image=ceph/daemon:latest
EOFconf

source cluster_conf.sh

cat << EOFconf >> cluster_conf.sh
ssh_slave_a="ssh root@${cmd_slave_a}"
ssh_slave_b="ssh root@${cmd_slave_b}"
EOFconf

source cluster_conf.sh

cat << EOFmon > start_mon.sh
# cat start_mon.sh
#!/bin/bash
docker run -d --net=host \
    --name=mon \
    --restart=always \
    -v /etc/localtime:/etc/localtime \
    -v /var/lib/docker/ceph_data/etc:/etc/ceph \
    -v /var/lib/docker/ceph_data/lib:/var/lib/ceph \
    -v /var/lib/docker/ceph_data/log:/var/log/ceph \
    -v /home/service/var:/var/lib/ceph/osd \
    -e MON_IP=${cmd_master},${cmd_slave_a},${cmd_slave_b} \
    -e CEPH_PUBLIC_NETWORK=${ceph_public_network} \
    ${ceph_daemon_image} mon
EOFmon

cat << EOFmds > start_mds.sh
#!/bin/bash
docker run -d \
    --net=host \
    --name=mds \
    --restart=always \
    --privileged=true \
    -v /etc/localtime:/etc/localtime \
    -v /var/lib/docker/ceph_data/etc:/etc/ceph \
    -v /var/lib/docker/ceph_data/lib:/var/lib/ceph \
    -v /var/lib/docker/ceph_data/log:/var/log/ceph \
    -e CEPHFS_CREATE=0 \
    -e CEPHFS_METADATA_POOL_PG=512 \
    -e CEPHFS_DATA_POOL_PG=512 \
    ${ceph_daemon_image} mds
EOFmds

cat << EOFmgr > start_mgr.sh
#!/bin/bash
docker run -d --net=host \
    --name=mgr \
    --restart=always \
    --privileged=true \
    -v /etc/localtime:/etc/localtime \
    -v /var/lib/docker/ceph_data/etc:/etc/ceph \
    -v /var/lib/docker/ceph_data/lib:/var/lib/ceph \
    -v /var/lib/docker/ceph_data/log:/var/log/ceph \
    ${ceph_daemon_image} mgr
EOFmgr

cat << EOFosd > start_osd.sh
#!/bin/bash

docker run -d \
--name=osd_data \
--net=host \
--restart=always \
--privileged=true \
--pid=host \
-v /etc/localtime:/etc/localtime \
-v /var/lib/docker/ceph_data/etc:/etc/ceph \
-v /var/lib/docker/ceph_data/lib:/var/lib/ceph \
-v /var/lib/docker/ceph_data/log:/var/log/ceph \
-v /home/service/var:/var/lib/ceph/osd \
-e OSD_TYPE=directory \
${ceph_daemon_image} osd
EOFosd

cat << EOFrgw > start_rgw.sh
#!/bin/bash
docker run -d \
    --net=host \
    --name=rgw \
    --restart=always \
    --privileged=true \
    -v /etc/localtime:/etc/localtime \
    -v /var/lib/docker/ceph_data/etc:/etc/ceph \
    -v /var/lib/docker/ceph_data/lib:/var/lib/ceph \
    -v /var/lib/docker/ceph_data/log:/var/log/ceph \
    ${ceph_daemon_image} rgw
EOFrgw

• /root/my_ceph_cluster/build.sh

#!/bin/bash

source cluster_conf.sh

echo ">>> rm container"
docker rm -f mon mgr osd_data mds rgw
${ssh_slave_a} docker rm -f mon mgr osd_data mds rgw
${ssh_slave_b} docker rm -f mon mgr osd_data mds rgw

echo ">>> cp script"
scp -r /root/my_ceph_cluster/ root@${cmd_slave_a}:/root/
scp -r /root/my_ceph_cluster/ root@${cmd_slave_b}:/root/

echo ">>> clean disk"
rm -rf /home/service/var/*
${ssh_slave_a} rm -rf /home/service/var/*
${ssh_slave_b} rm -rf /home/service/var/*

echo ">>> clean config/log file"
ceph_data_dir=/var/lib/docker/ceph_data/
rm -rf ${ceph_data_dir}
${ssh_slave_a} rm -rf ${ceph_data_dir}
${ssh_slave_b} rm -rf ${ceph_data_dir}

mkdir -p ${ceph_data_dir}/etc
mkdir -p ${ceph_data_dir}/lib
mkdir -p ${ceph_data_dir}/log
chmod -R 777 ${ceph_data_dir}


echo ">>> start local mon"
bash /root/my_ceph_cluster/start_mon.sh

sleep 5
docker exec mon ceph -s
docker exec mon ceph auth get client.bootstrap-osd -o /var/lib/ceph/bootstrap-osd/ceph.keyring

cat >>${ceph_data_dir}/etc/ceph.conf <<EOF
# 容忍更多的时钟误差
mon clock drift allowed = 2
mon clock drift warn backoff = 30
# 允许删除pool
mon_allow_pool_delete = true
mon_pg_warn_max_per_osd = 1000

# 每个节点存储大小
# 10G
bluestore_block_size = 10737418240
# 10T
# bluestore_block_size = 10995116277760
EOF

echo ">>> cp conf file"
scp -r /var/lib/docker/ceph_data root@${cmd_slave_a}:/var/lib/docker/
scp -r /var/lib/docker/ceph_data root@${cmd_slave_b}:/var/lib/docker/

sleep 2


echo ">>> start remote mon"
${ssh_slave_a} bash /root/my_ceph_cluster/start_mon.sh
${ssh_slave_b} bash /root/my_ceph_cluster/start_mon.sh

echo ">>> start mgr"
bash /root/my_ceph_cluster/start_mgr.sh
${ssh_slave_a} bash /root/my_ceph_cluster/start_mgr.sh
${ssh_slave_b} bash /root/my_ceph_cluster/start_mgr.sh

echo ">>> start osd"
sleep 5
bash /root/my_ceph_cluster/start_osd.sh
sleep 5
${ssh_slave_a} bash /root/my_ceph_cluster/start_osd.sh
sleep 5
${ssh_slave_b} bash /root/my_ceph_cluster/start_osd.sh
sleep 5

docker exec mon ceph -s

echo ">>> set data pool"
docker exec mon ceph osd pool create cephfs_data 128 128
sleep 5
docker exec mon ceph osd pool create cephfs_metadata 0 0
sleep 5
docker exec mon ceph fs new cephfs cephfs_metadata cephfs_data
sleep 5
docker exec mon ceph fs ls
sleep 3

echo ">>> start rgw"
docker exec mon ceph auth get client.bootstrap-rgw -o /var/lib/ceph/bootstrap-rgw/ceph.keyring
sleep 5
bash /root/my_ceph_cluster/start_rgw.sh
sleep 5
${ssh_slave_a} bash /root/my_ceph_cluster/start_rgw.sh
sleep 5
${ssh_slave_b} bash /root/my_ceph_cluster/start_rgw.sh
sleep 5

echo ">>> start mds"
bash /root/my_ceph_cluster/start_mds.sh
sleep 5
${ssh_slave_a} bash /root/my_ceph_cluster/start_mds.sh
sleep 5
${ssh_slave_b} bash /root/my_ceph_cluster/start_mds.sh
sleep 3

docker exec mon ceph -s
docker exec mon ceph fs ls
docker exec mon ceph mds stat
#docker exec mon ceph auth get client.bootstrap-rbd -o /var/lib/ceph/bootstrap-rbd-mirror/ceph.keyring

引用

• 使用Docker快速部署Ceph集群
• ceph documentation installation
• github ceph container readme
• 在Docker里运行Ceph
• 基于Docker部署ceph分布式文件系统(Luminous版本)
• CephFS 使用
• 挂载cephfs