Ceph安装

#sudo yum install -y yum-utils && sudo yum-config-manager --add-repo https://dl.fedoraproject.org/pub/epel/7/x86_64/ && sudo yum install --nogpgcheck -y epel-release && sudo rpm --import /etc/pki/rpm-gpg/RPM-GPG-KEY-EPEL-7 && sudo rm /etc/yum.repos.d/dl.fedoraproject.org*

sudo yum install yum-plugin-priorities

192.168.118.133 ceph-node0
192.168.118.134 ceph-node1
192.168.118.135 ceph-node2

机器名同步：
cat /etc/hosts
192.168.118.137 ceph-node0
192.168.118.138 ceph-node1
192.168.118.139 ceph-node2

防火墙
systemctl stop firewalld
systemctl disable firewalld

时间同步
yum install -y ntpdate

.安装epel源与ceph-deploy
wget -O /etc/yum.repos.d/epel.repo http://mirrors.aliyun.com/repo/epel-7.repo
安装ceph-deploy
rpm -ivh https://mirrors.tuna.tsinghua.edu.cn/ceph/rpm-luminous/el7/noarch/ceph-release-1-1.el7.noarch.rpm
替换 ceph.repo 服务器
sed -i ‘s#htt.*😕/download.ceph.com#https://mirrors.tuna.tsinghua.edu.cn/ceph#g’ /etc/yum.repos.d/ceph.repo

userdel ceph

adduser -d /home/ceph -m ceph
passwd ceph
1234567890

授权
echo “ceph ALL = (root) NOPASSWD:ALL” | sudo tee /etc/sudoers.d/ceph
chmod 0440 /etc/sudoers.d/ceph

ntp部署

ssh-keygen
ssh-copy-id ceph@ceph-node0
ssh-copy-id ceph@ceph-node1
ssh-copy-id ceph@ceph-node2

验证登录
ssh ‘ceph@ceph-node0’
ssh ‘ceph@ceph-node1’
ssh ‘ceph@ceph-node2’

下载源
sudo wget -O /etc/yum.repos.d/epel.repo http://mirrors.aliyun.com/repo/epel-7.repo

安装ceph-deploy
sudo rpm -ivh https://mirrors.tuna.tsinghua.edu.cn/ceph/rpm-luminous/el7/noarch/ceph-release-1-1.el7.noarch.rpm
替换
sudo sed -i ‘s#htt.*😕/download.ceph.com#https://mirrors.tuna.tsinghua.edu.cn/ceph#g’ /etc/yum.repos.d/ceph.repo

安装
sudo yum install -y ceph-deploy

安装后进入目录
mkdir ceph-install && cd ceph-install

ceph-deploy new ceph-node0 ceph-node1 ceph-node2

安装python2
yum install python2-pip* -y

安装
ceph-deploy install ceph-node0 ceph-node1 ceph-node2

或在每个节点上手动执行 yum install -y ceph

parted /dev/sdb mklabel gpt mkpart primary ext4 0% 50% mkpart primary ext4 50% 100% q

//防火墙
systemctl stop firewalld
systemctl disable firewalld

同步节点参数到各个节点
ceph-deploy --overwrite-conf mon create ceph{-node0,-node1,-node2}

ceph-deploy mon create-initial

1234567890

同步配置
ceph-deploy --overwrite-conf admin ceph-node0 ceph-node1 ceph-node2

元数据
ceph mds stat

yum provides ‘*/applydeltarpm’
sudo yum install deltarpm

查看集群监控状态
ceph mon_status

fdisk -l

查看端口
netstat -lntp

挂载osd
ceph-deploy osd create ceph-node2 --data /dev/sdb --journal /dev/sdb1
ceph-deploy osd create ceph-node1 --data /dev/sdb --journal /dev/sdb1

ceph -w

luminous 版本需要启动 mgr, 否则 ceph -s 会有 no active mgr 提示
每个monitor装mgr
ceph-deploy mgr create ceph-node0:ceph-node0 ceph-node1:ceph-node1 ceph-node2:ceph-node2

分离公共网络和集群网络(推荐、可选)

按下方所列修改配置文件 ceph.conf (在目录 ~/ceph-install 下操作，注意替换 fsid )
[global]

注意替换 fsid

fsid = dca70270-3292-4078-91c3-1fbefcd3bd62

mon_initial_members = ceph-node0, ceph-node1, ceph-node2
mon_host = 192.168.0.150,192.168.0.151,192.168.0.152
auth_cluster_required = cephx
auth_service_required = cephx
auth_client_required = cephx

public network = 192.168.0.0/24
cluster network = 172.16.1.0/24

[mon.a]
host = ceph-node0
mon addr = 192.168.0.150:6789

[mon.b]
host = ceph-node1
mon addr = 192.168.0.151:6789

[mon.c]
host = ceph-node2
mon addr = 192.168.0.152:6789

[osd]
osd data = /var/lib/ceph/osd/ceph-$id
osd journal size = 20000
osd mkfs type = xfs
osd mkfs options xfs = -f

filestore xattr use omap = true
filestore min sync interval = 10
filestore max sync interval = 15
filestore queue max ops = 25000
filestore queue max bytes = 10485760
filestore queue committing max ops = 5000
filestore queue committing max bytes = 10485760000

journal max write bytes = 1073714824
journal max write entries = 10000
journal queue max ops = 50000
journal queue max bytes = 10485760000

osd max write size = 512
osd client message size cap = 2147483648
osd deep scrub stride = 131072
osd op threads = 8
osd disk threads = 4
osd map cache size = 1024
osd map cache bl size = 128
osd mount options xfs = “rw,noexec,nodev,noatime,nodiratime,nobarrier”
osd recovery op priority = 4
osd recovery max active = 10
osd max backfills = 4

[client]
rbd cache = true
rbd cache size = 268435456
rbd cache max dirty = 134217728
rbd cache max dirty age = 5

同步配置
ceph-deploy --overwrite-conf admin ceph-node0 ceph-node1 ceph-node2

重启节点
systemctl restart ceph*.service ceph*.target

查看监听：
netstat -lntp
osd树查看
ceph osd tree

ceph-fs需要元数据mds服务
创建mds,osd上都创建，高可用性
ceph-deploy mds create ceph-node0 ceph-node1 ceph-node2

pool存储池：
ceph osd pool ls
创建存储池
ceph osd pool create data_data 32

查看已有文件系统
ceph fs ls

创建一个名称为data的文件系统
[root@ceph-node0 ceph-install]# ceph osd pool create data_data 32
[root@ceph-node0 ceph-install]# ceph osd pool create data_metadata 32
[root@ceph-node0 ceph-install]# ceph fs new data data_metadata data_data

使用客户端 ceph-fuse 挂载文件系统
[root@ceph-mon yum.repos.d]# yum install -y ceph-fuse

客户端创建目录挂载文件系统
// 创建data录目
[root@ceph-mon ~]# mkdir /data

// 挂载
root@ceph-mon ~]# ceph-fuse -m 192.168.118.137,192.168.118.138,192.168.118.139:6789 /data

查看pg状态
ceph pg stat

20 pgs: 20 active+undersized+degraded; 14512 kB data, 302 GB used, 6388 GB / 6691 GB avail; 12/36 objects degraded (33.333%)

查看集群监控状态
ceph health detail

创建安装ceph object gatway
格式：
$ ceph-deploy install --rgw [ …]
装在第2个节点

ceph-deploy install --rgw ceph-node0

管理RGW节点
Ceph CLI工具需要在管理员模式下运行，因此需要执行以下命令：

$ ceph-deploy admin ceph-node0

安装RGW实例
执行命令：

$ ceph-deploy rgw create ceph-node0

[root@ceph-node0 ceph-install]# ceph-deploy osd create ceph-node1 --data /dev/sdb1 --journal /dev/sdb1

ceph-deploy osd create ceph-node1 --data /dev/sdb1 --journal /home/osd

ceph-deploy osd prepare ceph-node1:/home/osd ceph-node2:/home/osd
ceph-deploy osd activate ceph-node1:/home/osd ceph-node2:/home/osd

ceph-deploy osd activate node2:/var/local/osd0 node3:/var/local/osd1

[root@ceph-node0 ceph-install]# ceph-deploy osd create ceph-node2 --data /dev/sdb1 --journal /dev/sdb1

ceph --admin-daemon /var/run/ceph/ceph-mon.ceph-node0.asok config set ceph-node0
ceph --admin-daemon /var/run/ceph/ceph-mon.ceph-node1.asok config set
ceph --admin-daemon /var/run/ceph/ceph-mon.ceph-node2.asok config set
public_network=192.168.118.0/255

清空数据
ceph-deploy purge ceph-node0 ceph-node1 ceph-node2
ceph-deploy purgedata ceph-node0 ceph-node1 ceph-node2
ceph-deploy forgetkeys && rm ceph.*

ceph-deploy purge node1 node2
ceph-deploy purgedata node1 node2
ceph-deploy forgetkeys && rm ceph.*

[Ceph-noarch]
name=Ceph noarch packages
baseurl=http://download.ceph.com/rpm-jewel/el7/noarch
enabled=1
gpgcheck=1
type=rpm-md
gpgkey=https://download.ceph.com/keys/release.asc
priority=1

parted /dev/sdb mklabel gpt mkpart primary xfs 0% 50% mkpart primary xfs 50% 100% q

磁盘分区挂载
fdisk /dev/sdb n – p --默认
查看
fdisk -l
创建文件系统
mke2fs -t ext4 /dev/sdb1 #ext4创建文件系统

内核重新读取分区表
partprobe /dev/sdb
建立ext4文件系统
mkfs.ext4 /dev/sdb1

挂载（临时挂载）
mount /dev/sdb1 /opt
mount /dev/sdb1 /home/osd
//永久挂载 修改配置文件 仿照上层
vim /etc/fstab

UUID的查询：　 blkid
/dev/sdb1: UUID=“d9405a25-9f13-4548-bc4b-b19d62c69126” TYPE=“ext4”

mount -a

2019.5.6-091311
创建S3用户

[ceph@ceph-node0 ceph-install]$ radosgw-admin user create --uid=“testuser” --display-name=“First User”
{
“user_id”: “testuser”,
“display_name”: “First User”,
“email”: “”,
“suspended”: 0,
“max_buckets”: 1000,
“auid”: 0,
“subusers”: [],
“keys”: [
{
“user”: “testuser”,
“access_key”: “46C4IXBZO8NQMDXWNSNV”,
“secret_key”: “WCLvYcbLGwviyyMv9G60kMs1lu6GvVYVrLY9qfPE”
}
],
“swift_keys”: [],
“caps”: [],
“op_mask”: “read, write, delete”,
“default_placement”: “”,
“placement_tags”: [],
“bucket_quota”: {
“enabled”: false,
“check_on_raw”: false,
“max_size”: -1,
“max_size_kb”: 0,
“max_objects”: -1
},
“user_quota”: {
“enabled”: false,
“check_on_raw”: false,
“max_size”: -1,
“max_size_kb”: 0,
“max_objects”: -1
},
“temp_url_keys”: [],
“type”: “rgw”,
“mfa_ids”: []
}

重启网关
sudo systemctl restart ceph-radosgw.service

错误
[ceph@ceph-node0 ceph-install]$ sudo ceph-deploy new ceph-node0 ceph-node1 ceph-node2
Traceback (most recent call last):
File “/bin/ceph-deploy”, line 18, in
from ceph_deploy.cli import main
File “/usr/lib/python2.7/site-packages/ceph_deploy/cli.py”, line 1, in
import pkg_resources
ImportError: No module named pkg_resource

[ceph_deploy.osd][DEBUG ] Host ceph-node2 is now ready for osd use.
[ceph@ceph-node0 ceph-install]$ ceph -s
2019-05-07 20:21:50.948 7fc7bbe71700 -1 auth: unable to find a keyring on /etc/ceph/ceph.client.admin.keyring,/etc/ceph/ceph.keyring,/etc/ceph/keyring,/etc/ceph/keyring.bin,: (2) No such file or directory
2019-05-07 20:21:50.948 7fc7bbe71700 -1 monclient: ERROR: missing keyring, cannot use cephx for authentication
[errno 2] error connecting to the cluster

解决方案：sudo chmod +r /etc/ceph/ceph.client.admin.keyring

原文地址：https://www.iteye.com/blog/tllyf-2440735