heketi-cli 常用命令

[root@hadoop03 glusterfs]# kubectl  get svc | grep heketi
deploy-heketi       ClusterIP   10.1.40.70   <none>        8080/TCP   11h
[root@hadoop03 kubernetes]# export HEKETI_CLI_SERVER=http://10.1.40.70:8080

[root@hadoop03 kubernetes]# heketi-cli topology info  --user admin --secret 'My Secret'

### 拓扑添加 ###
[root@hadoop03 kubernetes]# heketi-cli topology load --json=topology.json --user admin --secret 'My Secret' --server http://10.1.135.146:8080
        Found node hadoop01 on cluster 18ed8f6d4874cb5f4dd2325578e1417d
                Adding device /dev/sdb ... OK
        Found node hadoop02 on cluster 18ed8f6d4874cb5f4dd2325578e1417d
                Adding device /dev/sdb ... OK
        Found node hadoop03 on cluster 18ed8f6d4874cb5f4dd2325578e1417d
                Adding device /dev/sdb ... OK

[root@hadoop03 kubernetes]# heketi-cli cluster list --user admin --secret 'My Secret'
Clusters:
Id:a7903bf6436e2e3df3a1aa1591a5dee0 [file][block]

###
[root@hadoop03 kubernetes]# heketi-cli volume list --user admin --secret 'My Secret'
Id:257539465ea370c4ca7bd2f9a2c660d2    Cluster:a7903bf6436e2e3df3a1aa1591a5dee0    Name:vol_257539465ea370c4ca7bd2f9a2c660d2

###
[root@hadoop03 module]# heketi-cli  --server http://10.1.107.180:8080  --user admin --secret 'My Secret' volume list
Id:158e2b7138496265cd751e1fe568539b    Cluster:18ed8f6d4874cb5f4dd2325578e1417d    Name:heketidbstorage
Id:455b772c2fc9efcaac1b0b7e21aaa774    Cluster:18ed8f6d4874cb5f4dd2325578e1417d    Name:vol_455b772c2fc9efcaac1b0b7e21aaa774
### 
[root@hadoop03 module]# heketi-cli  --server http://10.1.107.180:8080  --user admin --secret 'My Secret' volume info 455b772c2fc9efcaac1b0b7e21aaa774
Name: vol_455b772c2fc9efcaac1b0b7e21aaa774
Size: 1
Volume Id: 455b772c2fc9efcaac1b0b7e21aaa774
Cluster Id: 18ed8f6d4874cb5f4dd2325578e1417d
Mount: 192.168.153.101:vol_455b772c2fc9efcaac1b0b7e21aaa774
Mount Options: backup-volfile-servers=192.168.153.103,192.168.153.102
Block: false
Free Size: 0
Reserved Size: 0
Block Hosting Restriction: (none)
Block Volumes: []
Durability Type: replicate
Distribute Count: 1
Replica Count: 3
Snapshot Factor: 1.00

###
[root@hadoop03 kubernetes]# heketi-cli node add --management-host-name hadoop01 --storage-host-name hadoop01 --user admin --secret 'My Secret' --cluster a7903bf6436e2e3df3a1aa1591a5dee0 --zone 1
Node information:
Id: 76a732511a03c5af1d3280ad1f545994
State: online
Cluster Id: a7903bf6436e2e3df3a1aa1591a5dee0
Zone: 1
Management Hostname hadoop01
Storage Hostname hadoop01



###
[root@hadoop03 kubernetes]# heketi-cli node list --user admin --secret 'My Secret'
Id:76a732511a03c5af1d3280ad1f545994     Cluster:a7903bf6436e2e3df3a1aa1591a5dee0
Id:a86e4c1eca1928988d9cd926857d0742     Cluster:a7903bf6436e2e3df3a1aa1591a5dee0
Id:c2ebaa92c7467ca407736850585c090a     Cluster:a7903bf6436e2e3df3a1aa1591a5dee0
[root@hadoop03 kubernetes]#

###
[root@hadoop03 kubernetes]# heketi-cli node info 76a732511a03c5af1d3280ad1f545994 --user admin --secret 'My Secret'
Node Id: 76a732511a03c5af1d3280ad1f545994
State: online
Cluster Id: a7903bf6436e2e3df3a1aa1591a5dee0
Zone: 1
Management Hostname: hadoop01
Storage Hostname: hadoop01
Devices:
[root@hadoop03 kubernetes]# heketi-cli node info a86e4c1eca1928988d9cd926857d0742 --user admin --secret 'My Secret'
Node Id: a86e4c1eca1928988d9cd926857d0742
State: online
Cluster Id: a7903bf6436e2e3df3a1aa1591a5dee0
Zone: 1
Management Hostname: hadoop03
Storage Hostname: 192.168.153.103
Devices:
Id:c373a6d564b78292f6f0a9f4b9a11a3e   Name:/dev/sdb            State:online    Size (GiB):4       Used (GiB):0       Free (GiB):4       Bricks:0    
[root@hadoop03 kubernetes]# heketi-cli device add --name=/dev/sdb --node=76a732511a03c5af1d3280ad1f545994  --user admin --secret 'My Secret'
Device added successfully
[root@hadoop03 kubernetes]# heketi-cli node info 76a732511a03c5af1d3280ad1f545994 --user admin --secret 'My Secret'
Node Id: 76a732511a03c5af1d3280ad1f545994
State: online
Cluster Id: a7903bf6436e2e3df3a1aa1591a5dee0
Zone: 1
Management Hostname: hadoop01
Storage Hostname: hadoop01
Devices:
Id:3ff795a77bd149da75558c510dcdbcb2   Name:/dev/sdb            State:online    Size (GiB):4       Used (GiB):0       Free (GiB):4       Bricks:0

####
[root@hadoop03 kubernetes]# heketi-cli setup-openshift-heketi-storage --user admin --secret 'My Secret'
Error: /usr/sbin/modprobe failed: 1
  thin: Required device-mapper target(s) not detected in your kernel.
  Run `lvcreate --help' for more information.
[root@hadoop03 kubernetes]# modprobe dm_thin_pool
[root@hadoop03 kubernetes]# heketi-cli setup-openshift-heketi-storage --user admin --secret 'My Secret'
Error: Failed to allocate new volume: Volume name 'heketidbstorage' already in use

### 
[root@hadoop03 kubernetes]# heketi-cli topology info  --user admin --secret 'My Secret'

扩容

[root@k8s helm]# kubectl  get pv
NAME                                       CAPACITY   ACCESS MODES   RECLAIM POLICY   STATUS   CLAIM                            STORAGECLASS     REASON   AGE
pvc-0419d8e3-cebe-4d45-b840-94fb24869283   1Gi        RWO            Delete           Bound    default/data-kafka-zookeeper-2   gluster-heketi            114d
pvc-b3ab31ce-55bb-47c6-9956-75e5dae1a308   1Gi        RWO            Delete           Bound    default/datadir-kafka-2          gluster-heketi            114d
pvc-b8d0e167-7b7f-496b-942d-4b8616fa0b02   1Gi        RWO            Delete           Bound    default/data-kafka-zookeeper-1   gluster-heketi            114d
pvc-c1451fdc-0fb2-41c9-ac25-0855bbf205a8   1Gi        RWO            Delete           Bound    default/datadir-kafka-0          gluster-heketi            114d
pvc-cc79a9eb-984b-4feb-9e23-75affaf38734   1Gi        RWO            Delete           Bound    default/data-kafka-zookeeper-0   gluster-heketi            114d
pvc-ff3ab399-ab2f-4e7f-8b0d-c60b5d6bd051   1Gi        RWO            Delete           Bound    default/datadir-kafka-1          gluster-heketi            114d
[root@k8s helm]# kubectl  describe pv pvc-cc79a9eb-984b-4feb-9e23-75affaf38734
Name:            pvc-cc79a9eb-984b-4feb-9e23-75affaf38734
Labels:          <none>
Annotations:     Description: Gluster-Internal: Dynamically provisioned PV
                 gluster.kubernetes.io/heketi-volume-id: 2b3e84581176001e4a123bcb7d3d681a
                 gluster.org/type: file
                 kubernetes.io/createdby: heketi-dynamic-provisioner
                 pv.beta.kubernetes.io/gid: 2000
                 pv.kubernetes.io/bound-by-controller: yes
                 pv.kubernetes.io/provisioned-by: kubernetes.io/glusterfs
Finalizers:      [kubernetes.io/pv-protection]
StorageClass:    gluster-heketi
Status:          Bound
Claim:           default/data-kafka-zookeeper-0
Reclaim Policy:  Delete
Access Modes:    RWO
VolumeMode:      Filesystem
Capacity:        1Gi
Node Affinity:   <none>
Message:         
Source:
    Type:                Glusterfs (a Glusterfs mount on the host that shares a pod's lifetime)
    EndpointsName:       glusterfs-dynamic-cc79a9eb-984b-4feb-9e23-75affaf38734
    EndpointsNamespace:  default
    Path:                vol_2b3e84581176001e4a123bcb7d3d681a
    ReadOnly:            false
Events:                  <none>

###
[root@k8s helm]# heketi-cli --user admin --secret 'My Secret' --server http://10.1.241.85:8080 volume expand --volume=2b3e84581176001e4a123bcb7d3d681a --expand-size=20
Name: vol_2b3e84581176001e4a123bcb7d3d681a
Size: 21
Volume Id: 2b3e84581176001e4a123bcb7d3d681a
Cluster Id: 563efc153a15e62560fd0afedd3a61fd
Mount: 10.0.2.35:vol_2b3e84581176001e4a123bcb7d3d681a
Mount Options: backup-volfile-servers=10.0.2.34,10.0.2.33
Block: false
Free Size: 0
Reserved Size: 0
Block Hosting Restriction: (none)
Block Volumes: []
Durability Type: replicate
Distribute Count: 2
Replica Count: 3
Snapshot Factor: 1.00



### 扩容前 ###
[root@k8s /]# gluster volume info vol_d09877fad932ec9130dc1306d69df036
 
Volume Name: vol_d09877fad932ec9130dc1306d69df036
Type: Replicate
Volume ID: 360a0f1d-fb4f-4b0f-a92a-16f6974363c7
Status: Started
Snapshot Count: 0
Number of Bricks: 1 x 3 = 3
Transport-type: tcp
Bricks:
Brick1: 10.0.2.33:/var/lib/heketi/mounts/vg_250a0c9d90faa2ad4b39b9dbe6c98866/brick_28b2464c652b8ea52457419994148e3c/brick
Brick2: 10.0.2.34:/var/lib/heketi/mounts/vg_108cece1b709b2d3e0cdebf35a2fec6d/brick_235d2c786ddae118a570283c036e2091/brick
Brick3: 10.0.2.35:/var/lib/heketi/mounts/vg_f47e1d26a2086bb2121f00512259f0e0/brick_e91e5299b467779f3ed6f988af78450b/brick
Options Reconfigured:
user.heketi.id: d09877fad932ec9130dc1306d69df036
transport.address-family: inet
nfs.disable: on
performance.client-io-threads: off

### 扩容后 ###
[root@k8s /]# gluster volume info vol_d09877fad932ec9130dc1306d69df036
 
Volume Name: vol_d09877fad932ec9130dc1306d69df036
Type: Distributed-Replicate
Volume ID: 360a0f1d-fb4f-4b0f-a92a-16f6974363c7
Status: Started
Snapshot Count: 0
Number of Bricks: 2 x 3 = 6
Transport-type: tcp
Bricks:
Brick1: 10.0.2.33:/var/lib/heketi/mounts/vg_250a0c9d90faa2ad4b39b9dbe6c98866/brick_28b2464c652b8ea52457419994148e3c/brick
Brick2: 10.0.2.34:/var/lib/heketi/mounts/vg_108cece1b709b2d3e0cdebf35a2fec6d/brick_235d2c786ddae118a570283c036e2091/brick
Brick3: 10.0.2.35:/var/lib/heketi/mounts/vg_f47e1d26a2086bb2121f00512259f0e0/brick_e91e5299b467779f3ed6f988af78450b/brick
Brick4: 10.0.2.34:/var/lib/heketi/mounts/vg_f38b5005f506abf6489e0bc70e603893/brick_340296705ff37d01b965668317a16329/brick
Brick5: 10.0.2.35:/var/lib/heketi/mounts/vg_cfb739be72d304f2d6ae1fa284baa9da/brick_07df74c681389e06606f7e700f3db2e2/brick
Brick6: 10.0.2.33:/var/lib/heketi/mounts/vg_298745840521498695d5437caa600bbc/brick_45bd1f256c288da2e14113006e3defef/brick
Options Reconfigured:
user.heketi.id: d09877fad932ec9130dc1306d69df036
transport.address-family: inet
nfs.disable: on
performance.client-io-threads: off
[root@k8s /]#