之前总感觉是在实验室里面做大数据,搞个3台服务器,每台服务器2T磁盘空间,感觉很容易驾驭,这次真的买了8台8*3750GB的服务器感觉,却有点无从下手了,买回来的是裸服务器,磁盘需要自己挂载。
rpm -qa |grep lvm2
yum install lvm2
fdisk -l | grep /dev/sd
pvcreate /dev/sda /dev/sdb /dev/sdc /dev/sdd /dev/sde /dev/sdf /dev/sdg /dev/sdh
pvdisplay
vgcreate vgdata /dev/sda /dev/sdb /dev/sdc /dev/sdd /dev/sde /dev/sdf /dev/sdg /dev/sdh
vgdisplay
当执行lvcreate
的时候,逻辑卷的大小成了犯难的事情。一台服务器29T,我设置多少个逻辑卷合适呢,最佳实践是什么呢?2T以下的参考LVM(逻辑卷管理),讲的比较清晰,2T以上就不行了。
[root@bigdata-0001 ~]# vgdisplay
--- Volume group ---
VG Name vgdata
System ID
Format lvm2
Metadata Areas 8
Metadata Sequence No 1
VG Access read/write
VG Status resizable
MAX LV 0
Cur LV 0
Open LV 0
Max PV 0
Cur PV 8
Act PV 8
VG Size <29.11 TiB
PE Size 4.00 MiB
Total PE 7630888
Alloc PE / Size 0 / 0
Free PE / Size 7630888 / <29.11 TiB
VG UUID Pviqvq-QYxa-YKjP-2Jej-h3OT-pugw-Ad6kk6
查阅资料Linux对大于2T的磁盘分区方法及创建逻辑卷LVM,原来linux分区有两种,当磁盘超过2.2T时,就不能采用fdisk进行分区,而应该采用Parted分区。
先将之前的操作回滚
vgremove vgdata
pvremove /dev/sda /dev/sdb /dev/sdc /dev/sdd /dev/sde /dev/sdf /dev/sdg /dev/sdh
pvscan
vgscan
查看初始状态
[root@bigdata-0003 ~]# parted -l
Error: /dev/sda: unrecognised disk label
Model: ATA ST4000NM0035-1V4 (scsi)
Disk /dev/sda: 4001GB
Sector size (logical/physical): 512B/512B
Partition Table: unknown
Disk Flags:
Error: /dev/sdb: unrecognised disk label
Model: ATA ST4000NM0035-1V4 (scsi)
Disk /dev/sdb: 4001GB
Sector size (logical/physical): 512B/512B
Partition Table: unknown
Disk Flags:
Error: /dev/sdc: unrecognised disk label
Model: ATA ST4000NM0035-1V4 (scsi)
Disk /dev/sdc: 4001GB
Sector size (logical/physical): 512B/512B
Partition Table: unknown
Disk Flags:
Error: /dev/sdd: unrecognised disk label
Model: ATA ST4000NM0035-1V4 (scsi)
Disk /dev/sdd: 4001GB
Sector size (logical/physical): 512B/512B
Partition Table: unknown
Disk Flags:
Error: /dev/sde: unrecognised disk label
Model: ATA ST4000NM0035-1V4 (scsi)
Disk /dev/sde: 4001GB
Sector size (logical/physical): 512B/512B
Partition Table: unknown
Disk Flags:
Error: /dev/sdf: unrecognised disk label
Model: ATA ST4000NM0035-1V4 (scsi)
Disk /dev/sdf: 4001GB
Sector size (logical/physical): 512B/512B
Partition Table: unknown
Disk Flags:
Error: /dev/sdg: unrecognised disk label
Model: ATA ST4000NM0035-1V4 (scsi)
Disk /dev/sdg: 4001GB
Sector size (logical/physical): 512B/512B
Partition Table: unknown
Disk Flags:
Error: /dev/sdh: unrecognised disk label
Model: ATA ST4000NM0035-1V4 (scsi)
Disk /dev/sdh: 4001GB
Sector size (logical/physical): 512B/512B
Partition Table: unknown
Disk Flags:
Model: Virtio Block Device (virtblk)
Disk /dev/vda: 42.9GB
Sector size (logical/physical): 512B/512B
Partition Table: msdos
Disk Flags:
Number Start End Size Type File system Flags
1 1049kB 42.9GB 42.9GB primary ext4 boot
[root@bigdata-0001 ~]# parted /dev/sda
GNU Parted 3.1
Using /dev/sda
Welcome to GNU Parted! Type 'help' to view a list of commands.
(parted) mklabel gpt
Warning: The existing disk label on /dev/sda will be destroyed and all data on this disk will be lost. Do you want to continue?
Yes/No? yes
(parted) mkpart primary ext4 0 -1
Warning: The resulting partition is not properly aligned for best performance.
Ignore/Cancel? i
(parted) print
Model: ATA ST4000NM0035-1V4 (scsi)
Disk /dev/sda: 4001GB
Sector size (logical/physical): 512B/512B
Partition Table: gpt
Disk Flags:
Number Start End Size File system Name Flags
1 17.4kB 4001GB 4001GB primary
(parted) q
Information: You may need to update /etc/fstab.
# 将磁盘格式为ext4
mkfs.ext4 /dev/sda
# 磁盘挂载
mkdir -p /dataa
mount /dev/sda /dataa
# 从下面可以看到磁盘空间有了
[root@bigdata-0001 ~]# df -lh
Filesystem Size Used Avail Use% Mounted on
devtmpfs 32G 0 32G 0% /dev
tmpfs 32G 0 32G 0% /dev/shm
tmpfs 32G 8.8M 32G 1% /run
tmpfs 32G 0 32G 0% /sys/fs/cgroup
/dev/vda1 40G 2.3G 35G 7% /
tmpfs 6.3G 0 6.3G 0% /run/user/0
/dev/sda 3.6T 89M 3.4T 1% /dataa
其他的磁盘按照这个思路执行
# 这里把-1改成4T,4T作为一整个磁盘
parted /dev/sdb mklabel gpt mkpart primary ext4 0 4T
mkfs.ext4 /dev/sdb -y
mkdir -p /datab
mount /dev/sdb /datab
# 这个时候看到了两个大空间
[root@bigdata-0001 ~]# df -lh
Filesystem Size Used Avail Use% Mounted on
devtmpfs 32G 0 32G 0% /dev
tmpfs 32G 0 32G 0% /dev/shm
tmpfs 32G 8.8M 32G 1% /run
tmpfs 32G 0 32G 0% /sys/fs/cgroup
/dev/vda1 40G 2.3G 35G 7% /
tmpfs 6.3G 0 6.3G 0% /run/user/0
/dev/sda 3.6T 89M 3.4T 1% /dataa
/dev/sdb 3.6T 89M 3.4T 1% /datab
参考parted创建磁盘分区并创建LVM(Linux合并多块大于2T的磁盘并合并到一个分区)这篇文档,可以看到它将多个磁盘合并成为一个逻辑分区,这样做目的是啥呢?大家都这样做吗?如果增加了磁盘,又将怎么办呢?是交给分布式系统,还是按照它的方式去操作呢,还需仔细考虑。
参考Linux LVM硬盘管理及LVM扩容,LVM将一个或多个硬盘的分区在逻辑上集合,相当于一个大硬盘来使用,当硬盘的空间不够使用的时候,可以继续将其它的硬盘的分区加入其中,这样可以实现磁盘空间的动态管理,相对于普通的磁盘分区有很大的灵活性。与传统的磁盘与分区相比,LVM为计算机提供了更高层次的磁盘存储。
这么看来,多个磁盘合成一个分区,是当前主流的最佳实践。所以方案要发生变化了,接下来实践一下,看如何合并
这里再次回滚
umount /dataa
umount /datab
rm -r /dataa
rm -r /datab
# 删掉分区,按照同样的方式把/dev/sdb也删掉
[root@bigdata-0001 ~]# parted /dev/sda
GNU Parted 3.1
Using /dev/sda
Welcome to GNU Parted! Type 'help' to view a list of commands.
(parted) toggle 1 lvm
Error: The flag 'lvm' is not available for loop disk labels.
(parted) rm
Partition number? 1
(parted) print
# 创建分区
[root@bigdata-0001 ~]# parted /dev/sda
GNU Parted 3.1
Using /dev/sda
Welcome to GNU Parted! Type 'help' to view a list of commands.
(parted) mklabel gpt
Warning: The existing disk label on /dev/sda will be destroyed and all data on this disk will be lost. Do you want to continue?
Yes/No? yes
(parted) mkpart primary ext4 0 -1
Warning: The resulting partition is not properly aligned for best performance.
Ignore/Cancel? i
(parted) toggle 1 lvm
(parted) q
Information: You may need to update /etc/fstab.
[root@bigdata-0001 ~]# partprobe
[root@bigdata-0001 ~]# pvcreate /dev/sda1
Physical volume "/dev/sda1" successfully created.
[root@bigdata-0001 ~]# pvscan
PV /dev/sda1 lvm2 [<3.64 TiB]
Total: 1 [<3.64 TiB] / in use: 0 [0 ] / in no VG: 1 [<3.64 TiB]
vgcreate vgdata /dev/sda1
[root@bigdata-0001 ~]# vgdisplay
--- Volume group ---
VG Name vgdata
System ID
Format lvm2
Metadata Areas 1
Metadata Sequence No 1
VG Access read/write
VG Status resizable
MAX LV 0
Cur LV 0
Open LV 0
Max PV 0
Cur PV 1
Act PV 1
VG Size <3.64 TiB
PE Size 4.00 MiB
Total PE 953861
Alloc PE / Size 0 / 0
Free PE / Size 953861 / <3.64 TiB
VG UUID BhdgNy-T9Xc-UZMs-tvDN-ImFC-7HgC-1ok5sS
在执行lvcreate -L 4001G -n lvData vgdata
,提示Volume group "vgdata" has insufficient free space (953861 extents): 1024256 required.
参考Linux LVM学习总结——Insufficient Free Extents for a Logical Volume,解决方案是
# 每个PE的大小是4M
953861*4/1024~=3726G
[root@bigdata-0001 ~]# lvcreate -L 3726G -n lvData vgdata
Logical volume "lvData" created.
[root@bigdata-0001 ~]# lvdisplay
--- Logical volume ---
LV Path /dev/vgdata/lvData
LV Name lvData
VG Name vgdata
LV UUID L5ZDpQ-qOF9-ZXUQ-iKZt-JQGl-YnBe-2pPeV6
LV Write Access read/write
LV Creation host, time bigdata-0001, 2020-12-25 23:10:13 +0800
LV Status available
# open 0
LV Size <3.64 TiB
Current LE 953856
Segments 1
Allocation inherit
Read ahead sectors auto
- currently set to 8192
Block device 252:0
mkfs.ext4 /dev/vgdata/lvData
mkdir /appdata
mount /dev/vgdata/lvData /appdata
[root@bigdata-0001 ~]# df -lh
Filesystem Size Used Avail Use% Mounted on
devtmpfs 32G 0 32G 0% /dev
tmpfs 32G 0 32G 0% /dev/shm
tmpfs 32G 8.9M 32G 1% /run
tmpfs 32G 0 32G 0% /sys/fs/cgroup
/dev/vda1 40G 2.3G 35G 7% /
tmpfs 6.3G 0 6.3G 0% /run/user/0
/dev/mapper/vgdata-lvData 3.6T 89M 3.4T 1% /appdata
# 开机自动挂载
echo "/dev/vgdata/lvData /appdata ext4 defaults 0 0" >> /etc/fstab
下面挂载第二块磁盘
parted /dev/sdb
mklabel gpt
mkpart primary ext4 0 -1
toggle 1 lvm
q
partprobe
# 普通磁盘转换成PV
pvcreate /dev/sdb1
[root@bigdata-0001 ~]# pvs
PV VG Fmt Attr PSize PFree
/dev/sda1 vgdata lvm2 a-- <3.64t 20.00m
/dev/sdb1 lvm2 --- <3.64t <3.64t
vgextend vgdata /dev/sdb1
[root@bigdata-0001 ~]# vgdisplay
--- Volume group ---
VG Name vgdata
System ID
Format lvm2
Metadata Areas 2
Metadata Sequence No 3
VG Access read/write
VG Status resizable
MAX LV 0
Cur LV 1
Open LV 1
Max PV 0
Cur PV 2
Act PV 2
VG Size <7.28 TiB
PE Size 4.00 MiB
Total PE 1907722
Alloc PE / Size 953856 / <3.64 TiB
Free PE / Size 953866 / <3.64 TiB
VG UUID BhdgNy-T9Xc-UZMs-tvDN-ImFC-7HgC-1ok5sS
# 扩容
lvextend -L +3726G /dev/vgdata/lvData
# resize2fs更新后才能识别
resize2fs /dev/vgdata/lvData
# 执行下面的命令,成功了
[root@bigdata-0001 ~]# df -lh
Filesystem Size Used Avail Use% Mounted on
devtmpfs 32G 0 32G 0% /dev
tmpfs 32G 0 32G 0% /dev/shm
tmpfs 32G 8.9M 32G 1% /run
tmpfs 32G 0 32G 0% /sys/fs/cgroup
/dev/vda1 40G 2.3G 35G 7% /
tmpfs 6.3G 0 6.3G 0% /run/user/0
/dev/mapper/vgdata-lvData 7.2T 51M 6.9T 1% /appdata
接下啦重复操作,这里最后在重新梳理,最终形成:
rpm -qa |grep lvm2
yum install lvm2 -y
# 查看有多少块盘
fdisk -l | grep /dev/sd
# 第一块磁盘
parted /dev/sda
mklabel gpt
mkpart primary ext4 0 -1
i
toggle 1 lvm
q
partprobe
pvcreate /dev/sda1
pvs
vgcreate vgdata /dev/sda1
vgdisplay
lvcreate -L 3726G -n lvData vgdata
mkfs.ext4 /dev/vgdata/lvData
mkdir /appdata
mount /dev/vgdata/lvData /appdata
echo "/dev/vgdata/lvData /appdata ext4 defaults 0 0" >> /etc/fstab
parted /dev/sdb
mklabel gpt
mkpart primary ext4 0 -1
i
toggle 1 lvm
q
partprobe
pvcreate /dev/sdb1
pvs
vgextend vgdata /dev/sdb1
vgdisplay
lvextend -L +3726G /dev/vgdata/lvData
resize2fs /dev/vgdata/lvData
# c块磁盘
parted /dev/sdc
mklabel gpt
mkpart primary ext4 0 -1
i
toggle 1 lvm
q
partprobe
pvcreate /dev/sdc1
pvs
vgextend vgdata /dev/sdc1
vgdisplay
# 3726G来自Free PE / Size 953871 / <3.64 TiB
# 953871*4/1024~=3726G
lvextend -L +3726G /dev/vgdata/lvData
resize2fs /dev/vgdata/lvData
parted /dev/sdd
mklabel gpt
mkpart primary ext4 0 -1
toggle 1 lvm
i
q
partprobe
pvcreate /dev/sdd1
pvs
vgextend vgdata /dev/sdd1
vgdisplay
lvextend -L +3726G /dev/vgdata/lvData
resize2fs /dev/vgdata/lvData
parted /dev/sde
mklabel gpt
mkpart primary ext4 0 -1
toggle 1 lvm
i
q
partprobe
pvcreate /dev/sde1
pvs
vgextend vgdata /dev/sde1
vgdisplay
lvextend -L +3726G /dev/vgdata/lvData
resize2fs /dev/vgdata/lvData
parted /dev/sdf
mklabel gpt
mkpart primary ext4 0 -1
i
toggle 1 lvm
q
partprobe
pvcreate /dev/sdf1
pvs
vgextend vgdata /dev/sdf1
vgdisplay
lvextend -L +3726G /dev/vgdata/lvData
resize2fs /dev/vgdata/lvData
parted /dev/sdg
mklabel gpt
mkpart primary ext4 0 -1
i
toggle 1 lvm
q
partprobe
pvcreate /dev/sdg1
pvs
vgextend vgdata /dev/sdg1
vgdisplay
lvextend -L +3726G /dev/vgdata/lvData
resize2fs /dev/vgdata/lvData
parted /dev/sdh
mklabel gpt
mkpart primary ext4 0 -1
i
toggle 1 lvm
q
partprobe
pvcreate /dev/sdh1
pvs
vgextend vgdata /dev/sdh1
vgdisplay
lvextend -L +3726G /dev/vgdata/lvData
resize2fs /dev/vgdata/lvData
df -lh