大数据服务器磁盘空间的分区

之前总感觉是在实验室里面做大数据，搞个3台服务器，每台服务器2T磁盘空间，感觉很容易驾驭，这次真的买了8台8*3750GB的服务器感觉，却有点无从下手了，买回来的是裸服务器，磁盘需要自己挂载。

rpm -qa |grep lvm2
yum install lvm2
fdisk -l | grep /dev/sd

pvcreate /dev/sda /dev/sdb /dev/sdc /dev/sdd /dev/sde /dev/sdf /dev/sdg /dev/sdh
pvdisplay

vgcreate vgdata /dev/sda /dev/sdb /dev/sdc /dev/sdd /dev/sde /dev/sdf /dev/sdg /dev/sdh
vgdisplay

当执行lvcreate的时候，逻辑卷的大小成了犯难的事情。一台服务器29T，我设置多少个逻辑卷合适呢，最佳实践是什么呢？2T以下的参考LVM（逻辑卷管理），讲的比较清晰，2T以上就不行了。

[root@bigdata-0001 ~]# vgdisplay
  --- Volume group ---
  VG Name               vgdata
  System ID             
  Format                lvm2
  Metadata Areas        8
  Metadata Sequence No  1
  VG Access             read/write
  VG Status             resizable
  MAX LV                0
  Cur LV                0
  Open LV               0
  Max PV                0
  Cur PV                8
  Act PV                8
  VG Size               <29.11 TiB
  PE Size               4.00 MiB
  Total PE              7630888
  Alloc PE / Size       0 / 0   
  Free  PE / Size       7630888 / <29.11 TiB
  VG UUID               Pviqvq-QYxa-YKjP-2Jej-h3OT-pugw-Ad6kk6

查阅资料Linux对大于2T的磁盘分区方法及创建逻辑卷LVM，原来linux分区有两种，当磁盘超过2.2T时，就不能采用fdisk进行分区，而应该采用Parted分区。
先将之前的操作回滚

vgremove vgdata
pvremove /dev/sda /dev/sdb /dev/sdc /dev/sdd /dev/sde /dev/sdf /dev/sdg /dev/sdh
pvscan
vgscan

查看初始状态

[root@bigdata-0003 ~]# parted -l
Error: /dev/sda: unrecognised disk label
Model: ATA ST4000NM0035-1V4 (scsi)                                        
Disk /dev/sda: 4001GB
Sector size (logical/physical): 512B/512B
Partition Table: unknown
Disk Flags: 

Error: /dev/sdb: unrecognised disk label
Model: ATA ST4000NM0035-1V4 (scsi)                                        
Disk /dev/sdb: 4001GB
Sector size (logical/physical): 512B/512B
Partition Table: unknown
Disk Flags: 

Error: /dev/sdc: unrecognised disk label
Model: ATA ST4000NM0035-1V4 (scsi)                                        
Disk /dev/sdc: 4001GB
Sector size (logical/physical): 512B/512B
Partition Table: unknown
Disk Flags: 

Error: /dev/sdd: unrecognised disk label
Model: ATA ST4000NM0035-1V4 (scsi)                                        
Disk /dev/sdd: 4001GB
Sector size (logical/physical): 512B/512B
Partition Table: unknown
Disk Flags: 

Error: /dev/sde: unrecognised disk label
Model: ATA ST4000NM0035-1V4 (scsi)                                        
Disk /dev/sde: 4001GB
Sector size (logical/physical): 512B/512B
Partition Table: unknown
Disk Flags: 

Error: /dev/sdf: unrecognised disk label
Model: ATA ST4000NM0035-1V4 (scsi)                                        
Disk /dev/sdf: 4001GB
Sector size (logical/physical): 512B/512B
Partition Table: unknown
Disk Flags: 

Error: /dev/sdg: unrecognised disk label
Model: ATA ST4000NM0035-1V4 (scsi)                                        
Disk /dev/sdg: 4001GB
Sector size (logical/physical): 512B/512B
Partition Table: unknown
Disk Flags: 

Error: /dev/sdh: unrecognised disk label
Model: ATA ST4000NM0035-1V4 (scsi)                                        
Disk /dev/sdh: 4001GB
Sector size (logical/physical): 512B/512B
Partition Table: unknown
Disk Flags: 

Model: Virtio Block Device (virtblk)
Disk /dev/vda: 42.9GB
Sector size (logical/physical): 512B/512B
Partition Table: msdos
Disk Flags: 

Number  Start   End     Size    Type     File system  Flags
1      1049kB  42.9GB  42.9GB  primary  ext4         boot

参考分区工具parted的详解及常用分区使用方法

[root@bigdata-0001 ~]# parted /dev/sda
GNU Parted 3.1
Using /dev/sda
Welcome to GNU Parted! Type 'help' to view a list of commands.
(parted) mklabel gpt                                                      
Warning: The existing disk label on /dev/sda will be destroyed and all data on this disk will be lost. Do you want to continue?
Yes/No? yes
(parted) mkpart primary ext4 0 -1                                         
Warning: The resulting partition is not properly aligned for best performance.
Ignore/Cancel? i                                                          
(parted) print                                                            
Model: ATA ST4000NM0035-1V4 (scsi)
Disk /dev/sda: 4001GB
Sector size (logical/physical): 512B/512B
Partition Table: gpt
Disk Flags: 

Number  Start   End     Size    File system  Name     Flags
1      17.4kB  4001GB  4001GB               primary

(parted) q                                                                
Information: You may need to update /etc/fstab.

# 将磁盘格式为ext4
mkfs.ext4 /dev/sda
# 磁盘挂载
mkdir -p /dataa
mount /dev/sda /dataa
# 从下面可以看到磁盘空间有了
[root@bigdata-0001 ~]# df -lh
Filesystem      Size  Used Avail Use% Mounted on
devtmpfs         32G     0   32G   0% /dev
tmpfs            32G     0   32G   0% /dev/shm
tmpfs            32G  8.8M   32G   1% /run
tmpfs            32G     0   32G   0% /sys/fs/cgroup
/dev/vda1        40G  2.3G   35G   7% /
tmpfs           6.3G     0  6.3G   0% /run/user/0
/dev/sda        3.6T   89M  3.4T   1% /dataa

其他的磁盘按照这个思路执行

# 这里把-1改成4T,4T作为一整个磁盘
parted /dev/sdb mklabel gpt mkpart primary ext4 0 4T
mkfs.ext4 /dev/sdb -y
mkdir -p /datab
mount /dev/sdb /datab
# 这个时候看到了两个大空间
[root@bigdata-0001 ~]# df -lh
Filesystem      Size  Used Avail Use% Mounted on
devtmpfs         32G     0   32G   0% /dev
tmpfs            32G     0   32G   0% /dev/shm
tmpfs            32G  8.8M   32G   1% /run
tmpfs            32G     0   32G   0% /sys/fs/cgroup
/dev/vda1        40G  2.3G   35G   7% /
tmpfs           6.3G     0  6.3G   0% /run/user/0
/dev/sda        3.6T   89M  3.4T   1% /dataa
/dev/sdb        3.6T   89M  3.4T   1% /datab

参考parted创建磁盘分区并创建LVM(Linux合并多块大于2T的磁盘并合并到一个分区)这篇文档，可以看到它将多个磁盘合并成为一个逻辑分区，这样做目的是啥呢？大家都这样做吗？如果增加了磁盘，又将怎么办呢？是交给分布式系统，还是按照它的方式去操作呢，还需仔细考虑。
参考Linux LVM硬盘管理及LVM扩容，LVM将一个或多个硬盘的分区在逻辑上集合，相当于一个大硬盘来使用，当硬盘的空间不够使用的时候，可以继续将其它的硬盘的分区加入其中，这样可以实现磁盘空间的动态管理，相对于普通的磁盘分区有很大的灵活性。与传统的磁盘与分区相比，LVM为计算机提供了更高层次的磁盘存储。
这么看来，多个磁盘合成一个分区，是当前主流的最佳实践。所以方案要发生变化了，接下来实践一下，看如何合并
这里再次回滚

umount /dataa
umount /datab
rm -r /dataa
rm -r /datab
# 删掉分区,按照同样的方式把/dev/sdb也删掉
[root@bigdata-0001 ~]# parted /dev/sda
GNU Parted 3.1
Using /dev/sda
Welcome to GNU Parted! Type 'help' to view a list of commands.
(parted) toggle 1 lvm
Error: The flag 'lvm' is not available for loop disk labels.
(parted) rm                                                               
Partition number? 1                                                       
(parted) print 

# 创建分区
[root@bigdata-0001 ~]# parted /dev/sda
GNU Parted 3.1
Using /dev/sda
Welcome to GNU Parted! Type 'help' to view a list of commands.
(parted) mklabel gpt                                                      
Warning: The existing disk label on /dev/sda will be destroyed and all data on this disk will be lost. Do you want to continue?
Yes/No? yes                                                               
(parted) mkpart primary ext4 0 -1                                         
Warning: The resulting partition is not properly aligned for best performance.
Ignore/Cancel? i                                                          
(parted) toggle 1 lvm                                                     
(parted) q                                                                
Information: You may need to update /etc/fstab.

[root@bigdata-0001 ~]# partprobe
[root@bigdata-0001 ~]# pvcreate /dev/sda1
  Physical volume "/dev/sda1" successfully created.
[root@bigdata-0001 ~]# pvscan
  PV /dev/sda1                      lvm2 [<3.64 TiB]
  Total: 1 [<3.64 TiB] / in use: 0 [0   ] / in no VG: 1 [<3.64 TiB]
  
vgcreate vgdata /dev/sda1
[root@bigdata-0001 ~]# vgdisplay
  --- Volume group ---
  VG Name               vgdata
  System ID             
  Format                lvm2
  Metadata Areas        1
  Metadata Sequence No  1
  VG Access             read/write
  VG Status             resizable
  MAX LV                0
  Cur LV                0
  Open LV               0
  Max PV                0
  Cur PV                1
  Act PV                1
  VG Size               <3.64 TiB
  PE Size               4.00 MiB
  Total PE              953861
  Alloc PE / Size       0 / 0   
  Free  PE / Size       953861 / <3.64 TiB
  VG UUID               BhdgNy-T9Xc-UZMs-tvDN-ImFC-7HgC-1ok5sS

在执行lvcreate -L 4001G -n lvData vgdata，提示Volume group "vgdata" has insufficient free space (953861 extents): 1024256 required.
参考Linux LVM学习总结——Insufficient Free Extents for a Logical Volume，解决方案是

# 每个PE的大小是4M
953861*4/1024~=3726G
[root@bigdata-0001 ~]# lvcreate -L 3726G -n lvData vgdata
  Logical volume "lvData" created.
[root@bigdata-0001 ~]# lvdisplay
  --- Logical volume ---
  LV Path                /dev/vgdata/lvData
  LV Name                lvData
  VG Name                vgdata
  LV UUID                L5ZDpQ-qOF9-ZXUQ-iKZt-JQGl-YnBe-2pPeV6
  LV Write Access        read/write
  LV Creation host, time bigdata-0001, 2020-12-25 23:10:13 +0800
  LV Status              available
  # open                 0
  LV Size                <3.64 TiB
  Current LE             953856
  Segments               1
  Allocation             inherit
  Read ahead sectors     auto
  - currently set to     8192
  Block device           252:0

mkfs.ext4 /dev/vgdata/lvData
mkdir /appdata
mount /dev/vgdata/lvData /appdata
[root@bigdata-0001 ~]# df -lh
Filesystem                 Size  Used Avail Use% Mounted on
devtmpfs                    32G     0   32G   0% /dev
tmpfs                       32G     0   32G   0% /dev/shm
tmpfs                       32G  8.9M   32G   1% /run
tmpfs                       32G     0   32G   0% /sys/fs/cgroup
/dev/vda1                   40G  2.3G   35G   7% /
tmpfs                      6.3G     0  6.3G   0% /run/user/0
/dev/mapper/vgdata-lvData  3.6T   89M  3.4T   1% /appdata

# 开机自动挂载
echo "/dev/vgdata/lvData /appdata ext4 defaults 0 0" >> /etc/fstab

下面挂载第二块磁盘

parted /dev/sdb
mklabel gpt 
mkpart primary ext4 0 -1
toggle 1 lvm
q
partprobe
# 普通磁盘转换成PV
pvcreate /dev/sdb1
[root@bigdata-0001 ~]# pvs
  PV         VG     Fmt  Attr PSize  PFree 
  /dev/sda1  vgdata lvm2 a--  <3.64t 20.00m
  /dev/sdb1         lvm2 ---  <3.64t <3.64t

vgextend vgdata /dev/sdb1
[root@bigdata-0001 ~]# vgdisplay
  --- Volume group ---
  VG Name               vgdata
  System ID             
  Format                lvm2
  Metadata Areas        2
  Metadata Sequence No  3
  VG Access             read/write
  VG Status             resizable
  MAX LV                0
  Cur LV                1
  Open LV               1
  Max PV                0
  Cur PV                2
  Act PV                2
  VG Size               <7.28 TiB
  PE Size               4.00 MiB
  Total PE              1907722
  Alloc PE / Size       953856 / <3.64 TiB
  Free  PE / Size       953866 / <3.64 TiB
  VG UUID               BhdgNy-T9Xc-UZMs-tvDN-ImFC-7HgC-1ok5sS

# 扩容
lvextend -L +3726G /dev/vgdata/lvData
# resize2fs更新后才能识别
resize2fs /dev/vgdata/lvData
# 执行下面的命令，成功了
[root@bigdata-0001 ~]# df -lh
Filesystem                 Size  Used Avail Use% Mounted on
devtmpfs                    32G     0   32G   0% /dev
tmpfs                       32G     0   32G   0% /dev/shm
tmpfs                       32G  8.9M   32G   1% /run
tmpfs                       32G     0   32G   0% /sys/fs/cgroup
/dev/vda1                   40G  2.3G   35G   7% /
tmpfs                      6.3G     0  6.3G   0% /run/user/0
/dev/mapper/vgdata-lvData  7.2T   51M  6.9T   1% /appdata

接下啦重复操作,这里最后在重新梳理，最终形成：

rpm -qa |grep lvm2
yum install lvm2 -y
# 查看有多少块盘
fdisk -l | grep /dev/sd

# 第一块磁盘
parted /dev/sda
mklabel gpt
mkpart primary ext4 0 -1
i
toggle 1 lvm
q
partprobe
pvcreate /dev/sda1
pvs
vgcreate vgdata /dev/sda1
vgdisplay
lvcreate -L 3726G -n lvData vgdata
mkfs.ext4 /dev/vgdata/lvData
mkdir /appdata
mount /dev/vgdata/lvData /appdata
echo "/dev/vgdata/lvData /appdata ext4 defaults 0 0" >> /etc/fstab

parted /dev/sdb
mklabel gpt
mkpart primary ext4 0 -1
i
toggle 1 lvm
q
partprobe
pvcreate /dev/sdb1
pvs
vgextend vgdata /dev/sdb1
vgdisplay
lvextend -L +3726G /dev/vgdata/lvData
resize2fs /dev/vgdata/lvData
# c块磁盘
parted /dev/sdc
mklabel gpt
mkpart primary ext4 0 -1
i
toggle 1 lvm
q
partprobe
pvcreate /dev/sdc1
pvs
vgextend vgdata /dev/sdc1
vgdisplay
# 3726G来自Free  PE / Size       953871 / <3.64 TiB
# 953871*4/1024~=3726G
lvextend -L +3726G /dev/vgdata/lvData
resize2fs /dev/vgdata/lvData

parted /dev/sdd
mklabel gpt
mkpart primary ext4 0 -1
toggle 1 lvm
i
q
partprobe
pvcreate /dev/sdd1
pvs
vgextend vgdata /dev/sdd1
vgdisplay
lvextend -L +3726G /dev/vgdata/lvData
resize2fs /dev/vgdata/lvData

parted /dev/sde
mklabel gpt
mkpart primary ext4 0 -1
toggle 1 lvm
i
q
partprobe
pvcreate /dev/sde1
pvs
vgextend vgdata /dev/sde1
vgdisplay
lvextend -L +3726G /dev/vgdata/lvData
resize2fs /dev/vgdata/lvData

parted /dev/sdf
mklabel gpt
mkpart primary ext4 0 -1
i
toggle 1 lvm
q
partprobe
pvcreate /dev/sdf1
pvs
vgextend vgdata /dev/sdf1
vgdisplay
lvextend -L +3726G /dev/vgdata/lvData
resize2fs /dev/vgdata/lvData

parted /dev/sdg
mklabel gpt
mkpart primary ext4 0 -1
i
toggle 1 lvm
q
partprobe
pvcreate /dev/sdg1
pvs
vgextend vgdata /dev/sdg1
vgdisplay
lvextend -L +3726G /dev/vgdata/lvData
resize2fs /dev/vgdata/lvData

parted /dev/sdh
mklabel gpt
mkpart primary ext4 0 -1
i
toggle 1 lvm
q
partprobe
pvcreate /dev/sdh1
pvs
vgextend vgdata /dev/sdh1
vgdisplay
lvextend -L +3726G /dev/vgdata/lvData
resize2fs /dev/vgdata/lvData
df -lh