Ceph初试

2019独角兽企业重金招聘Python工程师标准>>>

http://docs.ceph.org.cn/start

192.168.86.11   dbahost1 (admin-node)
192.168.86.16   dbahost6 (mon.node1、osd.2、mds.node1、rgw)
192.168.86.17   dbahost7 (mon.node2、osd.0)
192.168.86.18   dbahost8 (mon.node3、osd.1)

#安装 CEPH 部署工具ceph-deploy
wget -q -O- 'https://download.ceph.com/keys/release.asc' | sudo apt-key add -
echo deb http://download.eu.ceph.com/debian-luminous/ $(lsb_release -sc) main | sudo tee /etc/apt/sources.list.d/ceph.list
sudo apt-get update && sudo apt-get install ceph-deploy

#初始化环境
sudo useradd -d /home/cephadm -m cephadm
sudo passwd cephadm
---密码cephadm

echo "cephadm ALL = (root) NOPASSWD:ALL" | sudo tee /etc/sudoers.d/cephadm
sudo chmod 0440 /etc/sudoers.d/cephadm

在192.168.86.11上，生成sshkey-gen：
ssh-copy-id cephadm@192.168.86.16 
ssh-copy-id cephadm@192.168.86.17 
ssh-copy-id cephadm@192.168.86.18

192.168.86.11:

vi
/etc/hosts

192.168.86.11   dbahost1
192.168.86.16   dbahost6
192.168.86.17   dbahost7
192.168.86.18   dbahost8

~/.ssh/config

Host dbahost6
   Hostname dbahost6
   User cephadm
Host dbahost7
   Hostname dbahost7
   User cephadm
Host dbahost8
   Hostname dbahost8
   User cephadm
    

ssh dbahost1(在控制节点执行ceph-deploy):
部署初始monitor节点：
cd my-cluster
ceph-deploy new dbahost6

开始部署两个osd数据节点：
ceph-deploy install dbahost1 dbahost6 dbahost7 dbahost8

配置初始监控：
ceph-deploy mon create-initial

配置osd数据节点：
ssh dbahost7
sudo mkdir /var/local/osd0
sudo chmod a+w /var/local/osd0
exit

ssh dbahost8
sudo mkdir /var/local/osd1
sudo chmod a+w /var/local/osd1
exit

ssh dbahost1:

sudo vim ceph.conf
osd_max_object_name_len = 256
osd_max_object_namespace_len = 64

ceph-deploy osd prepare dbahost7:/var/local/osd0 dbahost8:/var/local/osd1
ceph-deploy osd activate dbahost7:/var/local/osd0 dbahost8:/var/local/osd1

ssh dbahost1:
ceph-deploy --overwrite-conf admin dbahost1 dbahost6 dbahost7 dbahost8
sudo chmod +r /etc/ceph/ceph.client.admin.keyring

开始部署扩容第3个osd数据节点：
ssh dbahost6
sudo mkdir /var/local/osd2
sudo chmod a+w /var/local/osd2

ssh dbahost1
ceph-deploy --overwrite-conf osd prepare dbahost6:/var/local/osd2
ceph-deploy osd activate dbahost6:/var/local/osd2

部署CephFS的元数据服务器:
ceph-deploy mds create dbahost6

部署ceph对象网关服务器radosgw：
ceph-deploy rgw create dbahost6

添加另外2个MONITORS：
ceph-deploy mon add dbahost7
ceph-deploy mon add dbahost8

sudo chmod -R a+r /etc/ceph
ceph quorum_status --format json-pretty

ceph -s
ceph -w
ceph osd tree
ceph osd lspools 
ceph fs ls

rbd ls
rbd showmapped
rbd map foo
rbd unmap foo

#ceph文件系统（就是一个NAS，网络文件系统）：
ceph osd pool create cephfs_data 50
ceph osd pool create cephfs_metadata 50
ceph fs new cephfs cephfs_metadata cephfs_data

在16上：
sudo mkdir -p /mnt/mycephfs1
sudo mount -t ceph 192.168.86.17:6789:/ /mnt/mycephfs1 -o name=admin,secret=AQDJGhRZAvkfMRAA2YzxketKJtF90jPh6InDDg==

在18上：
sudo mkdir -p /mnt/mycephfs2
sudo mount -t ceph 192.168.86.16:6789:/ /mnt/mycephfs2 -o name=admin,secret=AQDJGhRZAvkfMRAA2YzxketKJtF90jPh6InDDg==

结果，在16的/mnt/mycephfs1路径下的读写文件，在18上的/mnt/mycephfs2的路径下是能够看到和读写的，也就是二者共享了同一个网络卷。

#ceph块设备：
创建块设备的镜像：
rbd create foo --size 4096 -m 192.168.86.16 -k /etc/ceph/ceph.client.admin.keyring
rbd info foo
rbd feature disable foo exclusive-lock, object-map, fast-diff, deep-flatten

将镜像映射为块设备：
cephadm@dbahost1:/etc/ceph$ sudo rbd map foo --name client.admin -m 192.168.86.16 -k /etc/ceph/ceph.client.admin.keyring
/dev/rbd0
cephadm@dbahost1:/etc/ceph$ 

cephadm@dbahost1:/etc/ceph$ ls -l /dev/rbd/rbd/foo
lrwxrwxrwx 1 root root 10 May 12 10:10 /dev/rbd/rbd/foo -> ../../rbd0
cephadm@dbahost1:/etc/ceph$ ls -l /dev/rbd0
brw-rw---- 1 root disk 251, 0 May 12 10:10 /dev/rbd0
cephadm@dbahost1:/etc/ceph$ 

在块设备上创建文件系统（ext4）：
cephadm@dbahost1:/etc/ceph$ sudo mkfs.ext4 -m0 /dev/rbd/rbd/foo
mke2fs 1.42.13 (17-May-2015)
Discarding device blocks: done                            
Creating filesystem with 1048576 4k blocks and 262144 inodes
Filesystem UUID: 5f2b8485-4477-4e6e-a049-88dd5655f04c
Superblock backups stored on blocks: 
        32768, 98304, 163840, 229376, 294912, 819200, 884736

Allocating group tables: done                            
Writing inode tables: done                            
Creating journal (32768 blocks): done
Writing superblocks and filesystem accounting information: done 

cephadm@dbahost1:/etc/ceph$ 

挂载此文件系统：
sudo mkdir /mnt/ceph-block-device
sudo mount /dev/rbd/rbd/foo /mnt/ceph-block-device
cd /mnt/ceph-block-device

若其它节点再次挂载这个设备， 那么其它节点的卷是overlay的卷，可以看到这个卷原有的文件，新生成的文件互相都看不到。

#ceph对象存储
从管理节点的工作目录，在 client-node 上安装 Ceph 对象网关软件包:
ssh dbahost1
cd my-cluster
ceph-deploy install --rgw dbahost1
ceph-deploy rgw create dbahost1

可以从此url访问：
http://192.168.86.11:7480

systemctl restart ceph-radosgw.service

为 S3 访问创建 RADOSGW 用户：
cephadm@dbahost1:~/my-cluster$  sudo radosgw-admin user create --uid="testuser" --display-name="First User"
{
    "user_id": "testuser",
    "display_name": "First User",
    "email": "",
    "suspended": 0,
    "max_buckets": 1000,
    "auid": 0,
    "subusers": [],
    "keys": [
        {
            "user": "testuser",
            "access_key": "RBT0BVXHZ4OOFYSFKHG1",
            "secret_key": "
            "
        }
    ],
    "swift_keys": [],
    "caps": [],
    "op_mask": "read, write, delete",
    "default_placement": "",
    "placement_tags": [],
    "bucket_quota": {
        "enabled": false,
        "max_size_kb": -1,
        "max_objects": -1
    },
    "user_quota": {
        "enabled": false,
        "max_size_kb": -1,
        "max_objects": -1
    },
    "temp_url_keys": []
}

新建一个 SWIFT 子用户：
cephadm@dbahost1:~/my-cluster$ sudo radosgw-admin subuser create --uid=testuser --subuser=testuser:swift --access=full
{
    "user_id": "testuser",
    "display_name": "First User",
    "email": "",
    "suspended": 0,
    "max_buckets": 1000,
    "auid": 0,
    "subusers": [
        {
            "id": "testuser:swift",
            "permissions": "full-control"
        }
    ],
    "keys": [
        {
            "user": "testuser",
            "access_key": "RBT0BVXHZ4OOFYSFKHG1",
            "secret_key": "XXe5Ex1AzbkErBcRWC9IeaiZXzlDuLOlvpefCNu0"
        }
    ],
    "swift_keys": [
        {
            "user": "testuser:swift",
            "secret_key": "dQ2mMgQkqmwAyAZjjcgquoGN9I27OXN5hkk9bHcd"
        }
    ],
    "caps": [],
    "op_mask": "read, write, delete",
    "default_placement": "",
    "placement_tags": [],
    "bucket_quota": {
        "enabled": false,
        "max_size_kb": -1,
        "max_objects": -1
    },
    "user_quota": {
        "enabled": false,
        "max_size_kb": -1,
        "max_objects": -1
    },
    "temp_url_keys": []
}

cephadm@dbahost1:~/my-cluster$ 

新建 secret key:
cephadm@dbahost1:~/my-cluster$ sudo radosgw-admin key create --subuser=testuser:swift --key-type=swift --gen-secret
{
    "user_id": "testuser",
    "display_name": "First User",
    "email": "",
    "suspended": 0,
    "max_buckets": 1000,
    "auid": 0,
    "subusers": [
        {
            "id": "testuser:swift",
            "permissions": "full-control"
        }
    ],
    "keys": [
        {
            "user": "testuser",
            "access_key": "RBT0BVXHZ4OOFYSFKHG1",
            "secret_key": "XXe5Ex1AzbkErBcRWC9IeaiZXzlDuLOlvpefCNu0"
        }
    ],
    "swift_keys": [
        {
            "user": "testuser:swift",
            "secret_key": "w2TqR2vWNVCFUBapIOhAstxdoCW1lKkVt9Hyi7GE"
        }
    ],
    "caps": [],
    "op_mask": "read, write, delete",
    "default_placement": "",
    "placement_tags": [],
    "bucket_quota": {
        "enabled": false,
        "max_size_kb": -1,
        "max_objects": -1
    },
    "user_quota": {
        "enabled": false,
        "max_size_kb": -1,
        "max_objects": -1
    },
    "temp_url_keys": []
}

cephadm@dbahost1:~/my-cluster$ 

#测试s3方式的访问：
s3test.py ：
import boto
import boto.s3.connection

access_key = 'RBT0BVXHZ4OOFYSFKHG1'
secret_key = 'XXe5Ex1AzbkErBcRWC9IeaiZXzlDuLOlvpefCNu0'
conn = boto.connect_s3(
        aws_access_key_id = access_key,
        aws_secret_access_key = secret_key,
        host = '192.168.86.11', port = 7480 ,
        is_secure=False, calling_format = boto.s3.connection.OrdinaryCallingFormat(),
        )

bucket = conn.create_bucket('my-new-bucket')
    for bucket in conn.get_all_buckets():
            print "{name}".format(
                    name = bucket.name,
                    created = bucket.creation_date,
 )

@测试 SWIFT 访问

swift -A http://192.168.86.11:7480/auth/1.0 -U testuser:swift -K 'w2TqR2vWNVCFUBapIOhAstxdoCW1lKkVt9Hyi7GE' list

cephadm@dbahost1:~$ swift -A http://192.168.86.11:7480/auth/1.0 -U testuser:swift -K 'w2TqR2vWNVCFUBapIOhAstxdoCW1lKkVt9Hyi7GE' list
my-new-bucket
cephadm@dbahost1:~$ 

#结合docker的测试：

1、如果使用nfs即CephFS的文件系统，那么在不同host的不同容器内mount的卷， 其内容都是共享的，可以并发读写的。

192.168.86.18:6789:/  86302720  48271360  38031360  56% /mnt/mycephfs11
root@dbahost1:/mnt/mycephfs11# docker run -it --rm -v /mnt/mycephfs11:/mnt/mycephfs11 ubuntu /bin/bash 

root@c0c8aa7425a2:/mnt/mycephfs11# touch container_11
root@c0c8aa7425a2:/mnt/mycephfs11# ls -l 
total 1
-rw-r--r-- 1 root root  0 May 12 07:17 container_11
-rw-r--r-- 1 root root  0 May 12 07:16 dbahost1
-rw-r--r-- 1 root root 15 May 11 10:33 dbahost6
-rw-r--r-- 1 root root  0 May 12 07:17 dbahost7
root@c0c8aa7425a2:/mnt/mycephfs11#
root@c0c8aa7425a2:/mnt/mycephfs11# ls -l 
total 1
-rw-r--r-- 1 root root  0 May 12 07:17 container_11
-rw-r--r-- 1 root root  0 May 12 07:18 container_17
-rw-r--r-- 1 root root  0 May 12 07:16 dbahost1
-rw-r--r-- 1 root root 15 May 11 10:33 dbahost6
-rw-r--r-- 1 root root  0 May 12 07:17 dbahost7
root@c0c8aa7425a2:/mnt/mycephfs11# 

192.168.86.18:6789:/  86302720 48275456  38027264  56% /mnt/mycephfs3
root@dbahost7:/mnt/mycephfs3# docker run -it --rm -v /mnt/mycephfs3:/mnt/mycephfs17 ubuntu /bin/bash   

root@9d60c1af183c:/# df -k 
Filesystem           1K-blocks     Used Available Use% Mounted on
none                  28768380 14895136  12388856  55% /
tmpfs                   499152        0    499152   0% /dev
tmpfs                   499152        0    499152   0% /sys/fs/cgroup
192.168.86.18:6789:/  86302720 48275456  38027264  56% /mnt/mycephfs17
/dev/sda1             28768380 14895136  12388856  55% /etc/hosts
shm                      65536        0     65536   0% /dev/shm
tmpfs                   499152        0    499152   0% /sys/firmware
root@9d60c1af183c:/# cd  /mnt/mycephfs17 
root@9d60c1af183c:/mnt/mycephfs17# ls
container_11  dbahost1  dbahost6  dbahost7
root@9d60c1af183c:/mnt/mycephfs17# touch container_17
root@9d60c1af183c:/mnt/mycephfs17# ls -l 
total 1
-rw-r--r-- 1 root root  0 May 12 07:17 container_11
-rw-r--r-- 1 root root  0 May 12 07:18 container_17
-rw-r--r-- 1 root root  0 May 12 07:16 dbahost1
-rw-r--r-- 1 root root 15 May 11 10:33 dbahost6
-rw-r--r-- 1 root root  0 May 12 07:17 dbahost7
root@9d60c1af183c:/mnt/mycephfs17# 

2、如果使用ceph的块文件，那么在不同host的不同容器内mount的卷， 其内容不是共享的，是overlay的。
但是也可以用在容器共享上面， 因为后面一个容器的mount的卷里面的内容会包含该rbd设备的最新的内容。

http://ceph-users.ceph.narkive.com/LNCcfd0m/ceph-same-rbd-on-multiple-client

Example
create rbd image named foo
map foo to /dev/rbd0 on server A, mount /dev/rbd0 to /mnt
map foo to /dev/rbd0 on server B, mount /dev/rbd0 to /mnt

root@dbahost7:~# ls -l /dev/rbd/rbd/foo
lrwxrwxrwx 1 root root 10 May 12 10:20 /dev/rbd/rbd/foo -> ../../rbd0
root@dbahost7:~# 
root@dbahost7:~# 
root@dbahost7:~# mount /dev/rbd/rbd/foo /mnt/ceph-block-device-17
root@dbahost7:~# 
root@dbahost7:~# docker run -it --rm -v /mnt/ceph-block-device-17:/mnt/block-device17 ubuntu /bin/bash
root@e8c518538ff5:/# 
root@e8c518538ff5:/# df -k 
Filesystem     1K-blocks     Used Available Use% Mounted on
none            28768380 14896260  12387732  55% /
tmpfs             499152        0    499152   0% /dev
tmpfs             499152        0    499152   0% /sys/fs/cgroup
/dev/rbd0        3997376     8188   3972804   1% /mnt/block-device17
/dev/sda1       28768380 14896260  12387732  55% /etc/hosts
shm                65536        0     65536   0% /dev/shm
tmpfs             499152        0    499152   0% /sys/firmware
root@e8c518538ff5:/# cd /mnt/block-device17 
root@e8c518538ff5:/mnt/block-device17# echo "this is 17" > file17
root@e8c518538ff5:/mnt/block-device17# more file17
this is 17
root@e8c518538ff5:/mnt/block-device17# 

root@dbahost1:/mnt# ls -l /dev/rbd/rbd/foo
ls: cannot access '/dev/rbd/rbd/foo': No such file or directory
root@dbahost1:/mnt# 
root@dbahost1:/mnt# su - cephadm
cephadm@dbahost1:~$ sudo rbd map foo --name client.admin -m 192.168.86.16 -k /etc/ceph/ceph.client.admin.keyring
/dev/rbd0
cephadm@dbahost1:~$ ls -l /dev/rbd/rbd/foo 
lrwxrwxrwx 1 root root 10 May 12 15:58 /dev/rbd/rbd/foo -> ../../rbd0
cephadm@dbahost1:~$ sudo mount /dev/rbd/rbd/foo /mnt/ceph-block-device-11
cephadm@dbahost1:~$ df -k |grep device
/dev/rbd0              3997376      8188   3972804   1% /mnt/ceph-block-device-11
cephadm@dbahost1:~$ exit
logout
root@dbahost1:/mnt# docker run -it --rm -v /mnt/ceph-block-device-11:/mnt/block-device11 ubuntu /bin/bash
root@7f1d457f5ebd:/# df -k 
Filesystem     1K-blocks     Used Available Use% Mounted on
none            28768380 15316948  11967044  57% /
tmpfs            1015248        0   1015248   0% /dev
tmpfs            1015248        0   1015248   0% /sys/fs/cgroup
/dev/rbd0        3997376     8188   3972804   1% /mnt/block-device11
/dev/sda1       28768380 15316948  11967044  57% /etc/hosts
shm                65536        0     65536   0% /dev/shm
tmpfs            1015248        0   1015248   0% /sys/firmware
root@7f1d457f5ebd:/# cd /mnt/block-device11/
root@7f1d457f5ebd:/mnt/block-device11# ls
file17  lost+found
root@7f1d457f5ebd:/mnt/block-device11# more file17 
this is 17
root@7f1d457f5ebd:/mnt/block-device11# echo "this is 11" > file11
root@7f1d457f5ebd:/mnt/block-device11# ls -l
total 24
-rw-r--r-- 1 root root    11 May 12 07:59 file11
-rw-r--r-- 1 root root    11 May 12 07:57 file17
drwx------ 2 root root 16384 May 12 02:12 lost+found
root@7f1d457f5ebd:/mnt/block-device11# more file11
this is 11
root@7f1d457f5ebd:/mnt/block-device11# 

root@e8c518538ff5:/mnt/block-device17# ls -tlr 
total 20
drwx------ 2 root root 16384 May 12 02:12 lost+found
-rw-r--r-- 1 root root    11 May 12 07:57 file17
root@e8c518538ff5:/mnt/block-device17# 

转载于:https://my.oschina.net/rocky0202/blog/898319