1. 初始化环境
1.1 修改主机名
sudo hostnamectl set-hostname ceph-master1
1.2 修改网卡静态IP
这里根据个人环境选择是否配置静态IP。
cat | sudo tee /etc/systemd/network/eth1.network << EOF
[Match]
Name=eth1
[Network]
Address=10.10.10.254
EOF
1.3 使用阿里镜像对 Docker 拉取镜像进行加速
sudo mkdir /etc/systemd/system/docker.service.d
cat | sudo tee /etc/systemd/system/docker.service.d/registry-mirror.conf << EOF
[Service]
Environment=DOCKER_OPTS="--registry-mirror=https://8wzd8r7n.mirror.aliyuncs.com"
EOF
sudo systemctl daemon-reload && sudo systemctl restart docker
2. 启动管理组件
2.1 启动 MON 和 MGR 组件
# Ceph (MON & MGR)
docker run -d --restart=always --name mon --net=host -v /mnt/ceph/etc:/etc/ceph -v /mnt/ceph/lib:/var/lib/ceph -e MON_IP=10.10.10.254 -e CEPH_PUBLIC_NETWORK=10.10.10.0/24 ceph/daemon mon
docker run -d --restart=always --name mgr --net=host -v /mnt/ceph/etc:/etc/ceph -v /mnt/ceph/lib:/var/lib/ceph ceph/daemon mgr
2.2 修改配置文件
# Execute Scripts
cat | docker run --rm -i --privileged=true -v /mnt/ceph/etc:/etc/ceph -v /mnt/ceph/lib:/var/lib/ceph -v /dev/:/dev/ --entrypoint=bash ceph/daemon << EOF
# Config
echo osd_pool_default_size = 1 >> /etc/ceph/ceph.conf
echo [osd] >> /etc/ceph/ceph.conf
echo osd_client_message_size_cap = 10485760 >> /etc/ceph/ceph.conf
echo bluestore_cache_size = 104857600 >> /etc/ceph/ceph.conf
exit
EOF
3. 启动 OSD
3.1 初始化 OSD 使用的磁盘
删除原来磁盘存在 lvm
卷等信息,这里也需要根据自己个人环境选择是否执行。
# Execute Scripts
cat | docker run --rm -i --privileged=true -v /mnt/ceph/etc:/etc/ceph -v /mnt/ceph/lib:/var/lib/ceph -v /dev/:/dev/ --entrypoint=bash ceph/daemon << EOF
vgs | grep ceph | awk '{print \$1}' | xargs vgremove -y; pvremove /dev/sd{b,c,d}
exit
EOF
初始化磁盘 /dev/sdb
/dev/sdc
/dev/sdd
, 这里我使用3个磁盘分别创建了3个OSD。
# Execute Scripts
cat | docker run --rm -i --privileged=true -v /mnt/ceph/etc:/etc/ceph -v /mnt/ceph/lib:/var/lib/ceph -v /dev/:/dev/ --entrypoint=bash ceph/daemon << EOF
# OSD
ceph auth get client.bootstrap-osd -o /var/lib/ceph/bootstrap-osd/ceph.keyring
vgs | grep ceph | awk '{print \$1}' | xargs vgremove -y; pvremove /dev/sd{b,c,d}
ceph-volume lvm prepare --no-systemd --data /dev/sdb
ceph-volume lvm prepare --no-systemd --data /dev/sdc
ceph-volume lvm prepare --no-systemd --data /dev/sdd
exit
EOF
3.2 启动 OSD
启动上面初始化的3个OSD( /dev/sdb
/dev/sdc
/dev/sdd
)
docker run -d --restart=always --privileged=true --net=host --name osd.0 -v /mnt/ceph/etc:/etc/ceph -v /mnt/ceph/lib:/var/lib/ceph -v /dev/:/dev/ -e OSD_ID=0 ceph/daemon osd_ceph_volume_activate
docker run -d --restart=always --privileged=true --net=host --name osd.1 -v /mnt/ceph/etc:/etc/ceph -v /mnt/ceph/lib:/var/lib/ceph -v /dev/:/dev/ -e OSD_ID=1 ceph/daemon osd_ceph_volume_activate
docker run -d --restart=always --privileged=true --net=host --name osd.2 -v /mnt/ceph/etc:/etc/ceph -v /mnt/ceph/lib:/var/lib/ceph -v /dev/:/dev/ -e OSD_ID=2 ceph/daemon osd_ceph_volume_activate
4. 启动 MDS 并创建 RBD 镜像
4.1 启动 MDS
# Ceph (MDS)
docker run -d --restart=always --name mds --net=host -v /mnt/ceph/etc:/etc/ceph -v /mnt/ceph/lib:/var/lib/ceph -e CEPHFS_CREATE=1 ceph/daemon mds
4.2 创建并初始化 RBD 默认 Pool
# Execute Scripts
cat | docker run --rm -i -v /mnt/ceph/etc:/etc/ceph --entrypoint=bash ceph/daemon << EOF
ceph osd pool create rbd 100 100
rbd pool init
exit
EOF
4.3 创建 RBD 镜像
这里分别创建3个镜像 os-window_10
,data-develop_data
,data-develop_source
# Execute Scripts
cat | docker run --rm -i -v /mnt/ceph/etc:/etc/ceph --entrypoint=bash ceph/daemon << EOF
rbd create -s 128G os-window_10
rbd create -s 1024G data-develop_data
rbd create -s 1024G data-develop_source
exit
EOF
4.4 关闭RBD镜像特性
由于有些服务器自带的rbd
模块版本较低,可能不支持一些特性,所以这一步关闭掉新的特性来增加兼容性。
# Execute Scripts
cat | docker run --rm -i -v /mnt/ceph/etc:/etc/ceph --entrypoint=bash ceph/daemon << EOF
rbd feature disable os-window_10 exclusive-lock, object-map, fast-diff, deep-flatten
rbd feature disable data-develop_data exclusive-lock, object-map, fast-diff, deep-flatten
rbd feature disable data-develop_source exclusive-lock, object-map, fast-diff, deep-flatten
exit
EOF
5. 挂载 RBD 镜像到本地
5.1 获取挂载授权key
core@ceph-master1 ~ $ docker exec mon ceph auth export client.admin
export auth(key=AQCK2kxeUhZVHBAANZRWijI14HH/gjnBW+EXBw==)
[client.admin]
key = AQCK2kxeUhZVHBAANZRWijI14HH/gjnBW+EXBw==
caps mds = "allow *"
caps mgr = "allow *"
caps mon = "allow *"
caps osd = "allow *"
5.2 加载 RBD 模块并挂载
sudo modprobe rbd
sudo sh -c "echo 10.10.10.254 'name=admin,secret=AQCK2kxeUhZVHBAANZRWijI14HH/gjnBW+EXBw==' rbd data-develop_data > /sys/bus/rbd/add"
5.3 查看挂载的设备
core@localhost ~ $ ls -al /dev/rbd*
brw-rw----. 1 root disk 253, 0 Feb 20 13:08 /dev/rbd0