#####################
###1、安装ceph########
#####################
192.168.187.10 cf10 admin,osd,mon #管理和监控节点
192.168.187.20 cf20 osd,mds
192.168.187.30 cf30 osd,mds
192.168.187.40 cf40client client
osd节点，每个增加50G的磁盘，挂载为/storage
mkfs.xfs /dev/sdb
mount /dev/sdb /storage
修改挂载至/etc/fstab文件里面。
yum -y install vim net-tools lrzsz
systemctl disable firewalld
systemctl stop firewalld
systemctl disable NetworkManager
systemctl stop NetworkManager
禁止ipv6：
cat >> /etc/sysctl.conf <<EOF
net.ipv6.conf.all.disable_ipv6 =1
net.ipv6.conf.default.disable_ipv6 =1
EOF
sysctl -p
sed -i 's/SELINUX=enforcing/SELINUX=disabled/' /etc/sysconfig/selinux
sed -i 's/SELINUX=enforcing/SELINUX=disabled/' /etc/selinux/config
setenforce 0
cat>>/etc/hosts <<EOF
192.168.187.10 cf10
192.168.187.20 cf20
192.168.187.30 cf30
192.168.187.40 cf40client
EOF
在管理节点使用ssh-keygen 生成ssh keys 发布到各节点
[root@cf10 ~]# ssh-keygen
[root@cf10 ~]# ssh-copy-id cf10 #注意也要对自己免密钥认证
[root@cf10 ~]# ssh-copy-id cf20
[root@cf10 ~]# ssh-copy-id cf30
[root@cf10 ~]# ssh-copy-id cf40client
[root@xuegod67 ~]#vim /etc/yum.repos.d/ceph.repo
[Ceph]
name=Ceph packages for $basearch
baseurl=http://mirrors.aliyun.com/ceph/rpm-jewel/el7/x86_64
enabled=1
gpgcheck=1
type=rpm-md
gpgkey=http://mirrors.aliyun.com/ceph/keys/release.asc
priority=1
[Ceph-noarch]
name=Ceph noarch packages
baseurl=http://mirrors.aliyun.com/ceph/rpm-jewel/el7/noarch
enabled=1
gpgcheck=1
type=rpm-md
gpgkey=http://mirrors.aliyun.com/ceph/keys/release.asc
priority=1
[ceph-source]
name=Ceph source packages
baseurl=http://mirrors.aliyun.com/ceph/rpm-jewel/el7/SRPMS
enabled=1
gpgcheck=1
type=rpm-md
gpgkey=http://mirrors.aliyun.com/ceph/keys/release.asc
priority=1
yum clean all && yum list
yum -y install ceph-deploy
#在管理节点上创建mon服务
[root@cf10 ~]# mkdir /etc/ceph && cd /etc/ceph
[root@cf10 ceph]# ceph-deploy new cf10
[root@cf10 ceph]# ls -al
total 24
-rw-r--r--. 1 root root 196 Jul 31 11:18 ceph.conf #ceph配置文件
-rw-r--r--. 1 root root 2908 Jul 31 11:18 ceph-deploy-ceph.log #日志文件
-rw-------. 1 root root 73 Jul 31 11:18 ceph.mon.keyring #monitor密钥环
#修改副本的数量
[root@cf10 ceph]# cat ceph
cat: ceph: No such file or directory
[root@cf10 ceph]# cat ceph.conf
[global]
fsid = a3515ace-a5db-43de-a326-0a04fc4bc24b
mon_initial_members = cf10
mon_host = 192.168.187.10
auth_cluster_required = cephx
auth_service_required = cephx
auth_client_required = cephx
osd_pool_default_size = 2 #定义副本数量
#在所有节点上安装ceph,前提示每台服务都能联网yum源。
[root@cf10 ceph]# ceph-deploy install cf10 cf20 cf30 cf40client
#安装ceph的monitor服务
[root@cf10 ceph]# ceph-deploy mon create cf10
服务查看：
systemctl start ceph-mon@cf10
[root@cf10 ceph]# ceph --cluster=ceph --admin-daemon /var/run/ceph/ceph-mon.cf10.asok mon_status
[root@cf10 ceph]# ceph-deploy gatherkeys cf10 #收集节点的keyring文件
#创建osd，激活osd存储
[root@cf10 ceph]# ceph-deploy osd prepare cf10:/storage cf20:/storage cf30:/storage
[root@cf10 ceph]# ceph-deploy osd activate cf10:/storage cf20:/storage cf30:/storage
问题：
[cf10][WARNIN] 2019-07-31 15:41:36.845060 7f3407885ac0 -1 ** ERROR: error creating empty object store in /storage: (13) Permission denied
[cf10][WARNIN]
[cf10][ERROR ] RuntimeError: command returned non-zero exit status: 1
[ceph_deploy][ERROR ] RuntimeError: Failed to execute command: /usr/sbin/ceph-disk -v activate --mark-init systemd --mount /storage
解决如下：
chmod 777 -R /storage
#查看osd状态
[root@cf10 ceph]# ceph-deploy osd list cf10 cf20 cf30
#用ceph-deploy把配置文件和admin密钥拷贝到所有节点，这样每次执行Ceph命令行时就无需指定monitor地址和ceph.client.admin.keyring了
root@cf10 ceph]# ceph-deploy admin cf10 cf20 cf30
#各节点修改ceph.client.admin.keyring权限：
chmod +r /etc/ceph/ceph.client.admin.keyring
#查看集群状态
[root@cf30 ~]# ceph health
HEALTH_OK
[root@cf30 ~]# ceph -s
cluster a3515ace-a5db-43de-a326-0a04fc4bc24b
health HEALTH_OK
monmap e1: 1 mons at {cf10=192.168.187.10:6789/0}
election epoch 3, quorum 0 cf10
osdmap e14: 3 osds: 3 up, 3 in
flags sortbitwise,require_jewel_osds
pgmap v23: 64 pgs, 1 pools, 0 bytes data, 0 objects
15681 MB used, 134 GB / 149 GB avail
64 active+clean
#部署mds服务
[root@cf10 ceph]# ceph-deploy mds create cf20 cf30
[root@cf10 ceph]# ceph mds stat
#查看集群状态
[root@cf10 ceph]# ceph -s
cluster a3515ace-a5db-43de-a326-0a04fc4bc24b
health HEALTH_OK
monmap e1: 1 mons at {cf10=192.168.187.10:6789/0}
election epoch 3, quorum 0 cf10
osdmap e14: 3 osds: 3 up, 3 in
flags sortbitwise,require_jewel_osds
pgmap v23: 64 pgs, 1 pools, 0 bytes data, 0 objects
15681 MB used, 134 GB / 149 GB avail
64 active+clean
#以上基本上完成了ceph存储集群的搭建。
#####################
###2、创建文件系统####
####################
1、创建存储池
[root@cf10 ceph]# ceph fs ls
No filesystems enabled
[root@cf10 ceph]# ceph osd pool create cephfs_data 128 #128是pg_num
pool 'cephfs_data' created
[root@cf10 ceph]# ceph osd pool create cephfs_metadata 128 #128是pg_num
pool 'cephfs_metadata' created
确定 pg_num 取值是强制性的，因为不能自动计算。下面是几个常用的值：
*少于 5 个 OSD 时可把 pg_num 设置为 128
*OSD 数量在 5 到 10 个时，可把 pg_num 设置为 512
*OSD 数量在 10 到 50 个时，可把 pg_num 设置为 4096
*OSD 数量大于 50 时，你得理解权衡方法、以及如何自己计算 pg_num 取值
*自己计算 pg_num 取值时可借助 pgcalc 工具
随着 OSD 数量的增加，正确的 pg_num 取值变得更加重要，因为它显著地影响着集群的行为、以及出错时的数据持久性（即灾难性事件导致数据丢失的概率）。
2、创建文件系统
[root@cf10 ceph]# ceph fs new zgceph cephfs_metadata cephfs_data #其中zgceph可以自定义
new fs with metadata pool 2 and data pool 1
[root@cf10 ceph]#
new fs with metadata pool 2 and data pool 1
[root@cf10 ceph]# ceph fs ls
name: zgceph, metadata pool: cephfs_metadata, data pools: [cephfs_data ]
[root@cf10 ceph]# ceph mds stat
e6: 1/1/1 up {0=cf30=up:active}, 1 up:standby #active是活跃的，另1个是处于热备份的状态
[root@cf10 ceph]#
3、两种挂载方式，在客户端执行
注意：检查是否启用cephx认证方法，如果值为none为禁用，cephx为启用
[root@cf10 ceph]# cat /etc/ceph/ceph.conf | grep auth | grep required
auth_cluster_required = cephx
auth_service_required = cephx
auth_client_required = cephx
（1）使用内核mount挂载磁盘
从节点上拷贝密钥文件， cat /etc/ceph/ceph.client.admin.keyring，黏贴到如下的secret里面，注意去掉前面的用户名。
[root@cf40client ~]# cat admin.key
AQBrKUFd8KAaOxAAy1VmbpLtTq3DDlYdbHzV0g==
[root@cf40client ~]# mount -t ceph cf10:6789:/ /opt -o name=admin,secretfile=/etc/ceph/admin.secret
（2）使用ceph-fuse方式挂载
rpm -qa|grep -i ceph-fuse
[root@cf40client ~]# ceph-fuse --keyring /etc/ceph/ceph.client.admin.keyring --name client.admin -m cf10:6789 /opt
[root@cf40client ~]# cat /etc/ceph/ceph.client.admin.keyring
[client.admin]
key = AQBrKUFd8KAaOxAAy1VmbpLtTq3DDlYdbHzV0g==
#####################
###3、创建对象存储####
####################
1、使用命令行访问存储
[root@cf10 ceph]# rados mkpool test #创建对象池
successfully created pool test
[root@cf10 ceph]# rados rmpool test test --yes-i-really-really-mean-it #删除对象池及数据
successfully deleted pool test
[root@cf10 ceph]# rados create test-object -p zgoss
[root@cf10 ceph]# rados -p zgoss ls
test-object
[root@cf10 ceph]# rados rm test-object-1 -p zgoss
error removing zgoss>test-object-1: (2) No such file or directory
[root@cf10 ceph]# rados rm test-object -p zgoss
[root@cf10 ceph]#
[root@cf10 ceph]# rados -p zgoss put ceph ceph.conf
[root@cf10 ceph]# rados -p zgoss ls
ceph
test-object
[root@cf10 ~]# rados -p zgoss get ceph getfile #ceph是对象名称，getfile是输出的文件名。
2、使用s3cmd访问存储
[root@cf40client ~]# yum install s3cmd -y #在客户端安装工具
[root@cf10 ceph]# radosgw-admin user create --uid=radosgw --display-name="radosgw" #在节点端创建用户
{
"user_id": "radosgw",
"display_name": "radosgw",
"email": "",
"suspended": 0,
"max_buckets": 1000,
"auid": 0,
"subusers": [],
"keys": [
{
"user": "radosgw",
"access_key": "OKCCBQ92H3AVFH7IK51L", #注意密钥
"secret_key": "Ire9SOGcmM2iA21rnsOP3XEl5cBpWRDDzvnVV1vr" #注意密钥
}
],
"swift_keys": [],
"caps": [],
"op_mask": "read, write, delete",
"default_placement": "",
"placement_tags": [],
"bucket_quota": {
"enabled": false,
"max_size_kb": -1,
"max_objects": -1
},
"user_quota": {
"enabled": false,
"max_size_kb": -1,
"max_objects": -1
},
"temp_url_keys": []
}
[root@cf40client ~]# s3cmd --configure #创建s3配置文件
Enter new values or accept defaults in brackets with Enter.
Refer to user manual for detailed description of all options.
Access key and Secret key are your identifiers for Amazon S3. Leave them empty for using the env variables.
Access Key [OKCCBQ92H3AVFH7IK51L]:
Secret Key [Ire9SOGcmM2iA21rnsOP3XEl5cBpWRDDzvnVV1vr]:
Default Region [US]:
Use "s3.amazonaws.com" for S3 Endpoint and not modify it to the target Amazon S3.
S3 Endpoint [s3.amazonaws.com]:
Use "%(bucket)s.s3.amazonaws.com" to the target Amazon S3. "%(bucket)s" and "%(location)s" vars can be used
if the target S3 system supports dns based buckets.
DNS-style bucket+hostname:port template for accessing a bucket [%(bucket)s.s3.amazonaws.com]:
Encryption password is used to protect your files from reading
by unauthorized persons while in transfer to S3
Encryption password:
Path to GPG program [/usr/bin/gpg]:
When using secure HTTPS protocol all communication with Amazon S3
servers is protected from 3rd party eavesdropping. This method is
slower than plain HTTP, and can only be proxied with Python 2.7 or newer
Use HTTPS protocol [Yes]:
On some networks all internet access must go through a HTTP proxy.
Try setting it here if you can't connect to S3 directly
HTTP Proxy server name [n]: zgs3
HTTP Proxy server port [3128]:
New settings:
Access Key: OKCCBQ92H3AVFH7IK51L
Secret Key: Ire9SOGcmM2iA21rnsOP3XEl5cBpWRDDzvnVV1vr
Default Region: US
S3 Endpoint: s3.amazonaws.com
DNS-style bucket+hostname:port template for accessing a bucket: %(bucket)s.s3.amazonaws.com
Encryption password:
Path to GPG program: /usr/bin/gpg
Use HTTPS protocol: True
HTTP Proxy server name: zgs3
HTTP Proxy server port: 3128
Test access with supplied credentials? [Y/n] n
Save settings? [y/N] y
Configuration saved to '/root/.s3cfg'
[root@cf40client ~]#
[root@cf40client ~]# cat .s3cfg
[default]
access_key = OKCCBQ92H3AVFH7IK51L
access_token =
add_encoding_exts =
add_headers =
bucket_location = US
ca_certs_file =
cache_file =
check_ssl_certificate = True
check_ssl_hostname = True
cloudfront_host = cf10
content_disposition =
content_type =
default_mime_type = binary/octet-stream
delay_updates = False
delete_after = False
delete_after_fetch = False
delete_removed = False
dry_run = False
enable_multipart = True
encrypt = False
expiry_date =
expiry_days =
expiry_prefix =
follow_symlinks = False
force = False
get_continue = False
gpg_command = /usr/bin/gpg
gpg_decrypt = %(gpg_command)s -d --verbose --no-use-agent --batch --yes --passphrase-fd %(passphrase_fd)s -o %(output_file)s %(input_file)s
gpg_encrypt = %(gpg_command)s -c --verbose --no-use-agent --batch --yes --passphrase-fd %(passphrase_fd)s -o %(output_file)s %(input_file)s
gpg_passphrase =
guess_mime_type = True
host_base = cf10:6789
host_bucket = %(bucket)s.cf10
human_readable_sizes = False
invalidate_default_index_on_cf = False
invalidate_default_index_root_on_cf = True
invalidate_on_cf = False
kms_key =
limit = -1
limitrate = 0
list_md5 = False
log_target_prefix =
long_listing = False
max_delete = -1
mime_type =
multipart_chunk_size_mb = 15
multipart_max_chunks = 10000
preserve_attrs = True
progress_meter = True
proxy_host =
proxy_port = 0
put_continue = False
recursive = False
recv_chunk = 65536
reduced_redundancy = False
requester_pays = False
restore_days = 1
restore_priority = Standard
secret_key = Ire9SOGcmM2iA21rnsOP3XEl5cBpWRDDzvnVV1vr
send_chunk = 65536
server_side_encryption = False
signature_v2 = False
signurl_use_https = False
simpledb_host = sdb.amazonaws.com
skip_existing = False
socket_timeout = 300
stats = False
stop_on_error = False
storage_class =
throttle_max = 100
upload_id =
urlencoding_mode = normal
use_http_expect = False
use_https = False
use_mime_magic = True
verbosity = WARNING
website_endpoint = http://%(bucket)s.s3-website-%(location)s.amazonaws.com/
website_error =
website_index = index.html
#####################
###3、ceph集群运维####
####################
1、集群状态查看
[root@cf10 ceph]# ceph -s
cluster a3515ace-a5db-43de-a326-0a04fc4bc24b
health HEALTH_OK
monmap e1: 1 mons at {cf10=192.168.187.10:6789/0}
election epoch 3, quorum 0 cf10
fsmap e6: 1/1/1 up {0=cf30=up:active}, 1 up:standby
osdmap e19: 3 osds: 3 up, 3 in
flags sortbitwise,require_jewel_osds
pgmap v102: 320 pgs, 3 pools, 4202 bytes data, 20 objects
15682 MB used, 134 GB / 149 GB avail
320 active+clean
2、磁盘空间的使用情况
[root@cf10 ceph]# ceph df
GLOBAL:
SIZE AVAIL RAW USED %RAW USED
149G 134G 15682M 10.21
POOLS:
NAME ID USED %USED MAX AVAIL OBJECTS
rbd 0 0 0 65083M 0
cephfs_data 1 0 0 65083M 0
cephfs_metadata 2 4202 0 65083M 20
3、查看monitor状态
[root@cf10 ceph]# ceph mon stat
e1: 1 mons at {cf10=192.168.187.10:6789/0}, election epoch 3, quorum 0 cf10
4、查看mds状态
[root@cf10 ceph]# ceph mds stat
e6: 1/1/1 up {0=cf30=up:active}, 1 up:standby
[root@cf10 ceph]#
5、展示元数据集群的详细状态
[root@cf10 ceph]# ceph mds dump
dumped fsmap epoch 6
fs_name zgceph
epoch 6
flags 0
created 2019-07-31 16:00:27.800271
modified 2019-07-31 16:00:27.800271
tableserver 0
root 0
session_timeout 60
session_autoclose 300
max_file_size 1099511627776
last_failure 0
last_failure_osd_epoch 0
compat compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=file layout v2}
max_mds 1
in 0
up {0=4133}
failed
damaged
stopped
data_pools 1
metadata_pool 2
inline_data disabled
4133: 192.168.187.30:6804/2178842203 'cf30' mds.0.5 up:active seq 142
6、ceph集群中认证用户及相关的key
[root@cf10 ceph]# ceph auth list
installed auth entries:
mds.cf20
key: AQBnSEFdfp8sNhAAxYpbTvKgkm++W9MrbUobzw==
caps: [mds] allow
caps: [mon] allow profile mds
caps: [osd] allow rwx
mds.cf30
key: AQBpSEFdqPb0ExAAgR7eDNEz6GC6F279xAm/yg==
caps: [mds] allow
caps: [mon] allow profile mds
caps: [osd] allow rwx
osd.0
key: AQC9RkFdSq42MxAAEF2n1cdatqUvUJtCmguWdA==
caps: [mon] allow profile osd
caps: [osd] allow *
osd.1
key: AQDGRkFdnrsdHRAANn6dfD2P1xTZ33hOkhk0QA==
caps: [mon] allow profile osd
caps: [osd] allow *
osd.2
key: AQDPRkFdEExBJBAAzJVfs6Vl7lVu7BDYYRemvA==
caps: [mon] allow profile osd
caps: [osd] allow *
7、查看osd列表
[root@cf10 ceph]# ceph osd tree
ID WEIGHT TYPE NAME UP/DOWN REWEIGHT PRIMARY-AFFINITY
-1 0.14639 root default
-2 0.04880 host cf10
0 0.04880 osd.0 up 1.00000 1.00000
-3 0.04880 host cf20
1 0.04880 osd.1 up 1.00000 1.00000
-4 0.04880 host cf30
2 0.04880 osd.2 up 1.00000 1.00000
8、查看osd延迟信息
[root@cf10 ceph]# ceph osd perf
osd fs_commit_latency(ms) fs_apply_latency(ms)
2 11 147
1 9 134
0 11 84
9、查看OSD每块磁盘的空间
[root@cf10 ceph]# ceph osd df
ID WEIGHT REWEIGHT SIZE USE AVAIL %USE VAR PGS
0 0.04880 1.00000 51175M 5227M 45947M 10.21 1.00 191
1 0.04880 1.00000 51175M 5227M 45947M 10.21 1.00 220
2 0.04880 1.00000 51175M 5227M 45947M 10.21 1.00 229
TOTAL 149G 15682M 134G 10.21
MIN/MAX VAR: 1.00/1.00 STDDEV: 0
10、查看pool信息
[root@cf10 ceph]# ceph osd lspools
0 rbd,1 cephfs_data,2 cephfs_metadata,
[root@cf10 ceph]# ceph osd pool ls
rbd
cephfs_data
cephfs_metadata
##########################
###3、ceph集群开关机服务####
##########################
#先关闭ceph上的虚拟机
ceph osd set noout
ceph osd set norecover
ceph osd set norebalance
ceph osd set nobackfill
ceph osd set nodown
ceph osd set pause
先关闭OSD节点 再关闭monitor节点
开机
先开monitor节点 再开osd节点
所有osd都in后
取消标签
ceph osd unset noout
ceph osd unset norecover
ceph osd unset norebalance
ceph osd unset nobackfill
ceph osd unset nodown
ceph osd unset pause
#node节点信息
[root@cf10 ~]# systemctl list-unit-files|grep ceph
ceph-create-keys@.service static
ceph-disk@.service static
ceph-mds@.service disabled
ceph-mon@.service enabled
ceph-osd@.service enabled
ceph-radosgw@.service disabled
ceph-mds.target enabled
ceph-mon.target enabled
ceph-osd.target enabled
ceph-radosgw.target enabled
ceph.target enabled
1516
被折叠的 条评论
为什么被折叠?
到【灌水乐园】发言
