10.10.10.10 master
10.10.20.20 slave
10.10.10.30 slave
1. 均安装了 docker
2. ceph 数据盘挂载在 /home/service/var
3. 目录存在 /var/lib/docker/
- 设置 master 通过 ssh 免密登陆 slaves
ssh-keygen
modify slave /etc/hosts.allow
master .ssh/id_rsa.pub >> slave .ssh/authorized_keys
ssh root@10.10.20.20
ssh root@10.10.30.30
# 在 master 机器上
cd /root/my_ceph_cluster/
bash setup.sh
bash build.sh
# master
cat /var/lib/docker/ceph_data/etc/ceph.client.admin.keyring
# 测试机器上
mkdir /mnt/cephfs
mount -t ceph 10.10.10.10:6789,10.10.20.20:6789,10.10.30.30:6789:/ /mnt/cephfs/ -o name=admin,secret=xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx=
dd if=/dev/zero of=test_zero bs=1M count=1024
dd if=/dev/urandom of=test_random bs=1M count=1024
脚本
- /root/my_ceph_cluster/setup.sh
#!/bin/bash
cat << EOFconf > cluster_conf.sh
cmd_master=10.10.10.10
cmd_slave_a=10.10.20.20
cmd_slave_b=10.10.30.30
ceph_public_network=10.10.10.0/24,10.10.20.0/24,10.10.30.0/24
ceph_daemon_image=ceph/daemon:latest
EOFconf
source cluster_conf.sh
cat << EOFconf >> cluster_conf.sh
ssh_slave_a="ssh root@${cmd_slave_a}"
ssh_slave_b="ssh root@${cmd_slave_b}"
EOFconf
source cluster_conf.sh
cat << EOFmon > start_mon.sh
# cat start_mon.sh
#!/bin/bash
docker run -d --net=host \
--name=mon \
--restart=always \
-v /etc/localtime:/etc/localtime \
-v /var/lib/docker/ceph_data/etc:/etc/ceph \
-v /var/lib/docker/ceph_data/lib:/var/lib/ceph \
-v /var/lib/docker/ceph_data/log:/var/log/ceph \
-v /home/service/var:/var/lib/ceph/osd \
-e MON_IP=${cmd_master},${cmd_slave_a},${cmd_slave_b} \
-e CEPH_PUBLIC_NETWORK=${ceph_public_network} \
${ceph_daemon_image} mon
EOFmon
cat << EOFmds > start_mds.sh
#!/bin/bash
docker run -d \
--net=host \
--name=mds \
--restart=always \
--privileged=true \
-v /etc/localtime:/etc/localtime \
-v /var/lib/docker/ceph_data/etc:/etc/ceph \
-v /var/lib/docker/ceph_data/lib:/var/lib/ceph \
-v /var/lib/docker/ceph_data/log:/var/log/ceph \
-e CEPHFS_CREATE=0 \
-e CEPHFS_METADATA_POOL_PG=512 \
-e CEPHFS_DATA_POOL_PG=512 \
${ceph_daemon_image} mds
EOFmds
cat << EOFmgr > start_mgr.sh
#!/bin/bash
docker run -d --net=host \
--name=mgr \
--restart=always \
--privileged=true \
-v /etc/localtime:/etc/localtime \
-v /var/lib/docker/ceph_data/etc:/etc/ceph \
-v /var/lib/docker/ceph_data/lib:/var/lib/ceph \
-v /var/lib/docker/ceph_data/log:/var/log/ceph \
${ceph_daemon_image} mgr
EOFmgr
cat << EOFosd > start_osd.sh
#!/bin/bash
docker run -d \
--name=osd_data \
--net=host \
--restart=always \
--privileged=true \
--pid=host \
-v /etc/localtime:/etc/localtime \
-v /var/lib/docker/ceph_data/etc:/etc/ceph \
-v /var/lib/docker/ceph_data/lib:/var/lib/ceph \
-v /var/lib/docker/ceph_data/log:/var/log/ceph \
-v /home/service/var:/var/lib/ceph/osd \
-e OSD_TYPE=directory \
${ceph_daemon_image} osd
EOFosd
cat << EOFrgw > start_rgw.sh
#!/bin/bash
docker run -d \
--net=host \
--name=rgw \
--restart=always \
--privileged=true \
-v /etc/localtime:/etc/localtime \
-v /var/lib/docker/ceph_data/etc:/etc/ceph \
-v /var/lib/docker/ceph_data/lib:/var/lib/ceph \
-v /var/lib/docker/ceph_data/log:/var/log/ceph \
${ceph_daemon_image} rgw
EOFrgw
- /root/my_ceph_cluster/build.sh
#!/bin/bash
source cluster_conf.sh
echo ">>> rm container"
docker rm -f mon mgr osd_data mds rgw
${ssh_slave_a} docker rm -f mon mgr osd_data mds rgw
${ssh_slave_b} docker rm -f mon mgr osd_data mds rgw
echo ">>> cp script"
scp -r /root/my_ceph_cluster/ root@${cmd_slave_a}:/root/
scp -r /root/my_ceph_cluster/ root@${cmd_slave_b}:/root/
echo ">>> clean disk"
rm -rf /home/service/var/*
${ssh_slave_a} rm -rf /home/service/var/*
${ssh_slave_b} rm -rf /home/service/var/*
echo ">>> clean config/log file"
ceph_data_dir=/var/lib/docker/ceph_data/
rm -rf ${ceph_data_dir}
${ssh_slave_a} rm -rf ${ceph_data_dir}
${ssh_slave_b} rm -rf ${ceph_data_dir}
mkdir -p ${ceph_data_dir}/etc
mkdir -p ${ceph_data_dir}/lib
mkdir -p ${ceph_data_dir}/log
chmod -R 777 ${ceph_data_dir}
echo ">>> start local mon"
bash /root/my_ceph_cluster/start_mon.sh
sleep 5
docker exec mon ceph -s
docker exec mon ceph auth get client.bootstrap-osd -o /var/lib/ceph/bootstrap-osd/ceph.keyring
cat >>${ceph_data_dir}/etc/ceph.conf <<EOF
# 容忍更多的时钟误差
mon clock drift allowed = 2
mon clock drift warn backoff = 30
# 允许删除pool
mon_allow_pool_delete = true
mon_pg_warn_max_per_osd = 1000
# 每个节点存储大小
# 10G
bluestore_block_size = 10737418240
# 10T
# bluestore_block_size = 10995116277760
EOF
echo ">>> cp conf file"
scp -r /var/lib/docker/ceph_data root@${cmd_slave_a}:/var/lib/docker/
scp -r /var/lib/docker/ceph_data root@${cmd_slave_b}:/var/lib/docker/
sleep 2
echo ">>> start remote mon"
${ssh_slave_a} bash /root/my_ceph_cluster/start_mon.sh
${ssh_slave_b} bash /root/my_ceph_cluster/start_mon.sh
echo ">>> start mgr"
bash /root/my_ceph_cluster/start_mgr.sh
${ssh_slave_a} bash /root/my_ceph_cluster/start_mgr.sh
${ssh_slave_b} bash /root/my_ceph_cluster/start_mgr.sh
echo ">>> start osd"
sleep 5
bash /root/my_ceph_cluster/start_osd.sh
sleep 5
${ssh_slave_a} bash /root/my_ceph_cluster/start_osd.sh
sleep 5
${ssh_slave_b} bash /root/my_ceph_cluster/start_osd.sh
sleep 5
docker exec mon ceph -s
echo ">>> set data pool"
docker exec mon ceph osd pool create cephfs_data 128 128
sleep 5
docker exec mon ceph osd pool create cephfs_metadata 0 0
sleep 5
docker exec mon ceph fs new cephfs cephfs_metadata cephfs_data
sleep 5
docker exec mon ceph fs ls
sleep 3
echo ">>> start rgw"
docker exec mon ceph auth get client.bootstrap-rgw -o /var/lib/ceph/bootstrap-rgw/ceph.keyring
sleep 5
bash /root/my_ceph_cluster/start_rgw.sh
sleep 5
${ssh_slave_a} bash /root/my_ceph_cluster/start_rgw.sh
sleep 5
${ssh_slave_b} bash /root/my_ceph_cluster/start_rgw.sh
sleep 5
echo ">>> start mds"
bash /root/my_ceph_cluster/start_mds.sh
sleep 5
${ssh_slave_a} bash /root/my_ceph_cluster/start_mds.sh
sleep 5
${ssh_slave_b} bash /root/my_ceph_cluster/start_mds.sh
sleep 3
docker exec mon ceph -s
docker exec mon ceph fs ls
docker exec mon ceph mds stat
#docker exec mon ceph auth get client.bootstrap-rbd -o /var/lib/ceph/bootstrap-rbd-mirror/ceph.keyring
引用