NFS配置及优化

NFS配置

1. 不同的内容服务采用不同的文件系统，文件类型的可以是ext4，数据库类型的可以是xfs
2. linux系统的安装，多硬盘，系统盘和数据盘分开，使用lvm方式，数据盘大小要一致（不一致导致存储浪费）
3. 完成NFS 服务的安装，不同的Linux版本不一样，具体的安装方法可以baidu或者google，进行磁盘分区，如果使用ssd可以使用lvm缓存机制
4. 条带化创建vg

5. /boot需要单独的物理分区设置

   pvcreate针对现有的所有的硬盘

   vgcreate或者是vgextende 现有的vg

6. lvcreate -i 3（pv个数） -I 64 -l 1533 -n stripe_lv vg01
   lvcreate -L 50G -i2 -I64 -n gfslv vg0
   
   lvcreate --extents 100%FREE --stripes 5 --stripesize 256 --name nfs_lv ubuntu-vg
   
   扩展
   pvcreate之后，vgextend, 再lvextend
   
   http://tldp.org/HOWTO/LVM-HOWTO/recipethreescsistripe.html
   https://blog.csdn.net/cybertan/article/details/6773214
   
   vgextend
   lvextend -l +100%FREE  /dev/ubuntu-vg/nfs_lv
   resize2fs -p -F /dev/mapper/ubuntu--vg-nfs_lv
7. linux disk 优化
   NSF 服务优化
   磁盘调度算法
   echo deadline > /sys/block/sda/queue/scheduler
   echo deadline > /sys/block/sdb/queue/scheduler
   磁盘队列
   echo 512 > /sys/block/sda/queue/nr_requests
   echo 512 > /sys/block/sdb/queue/nr_requests
   预读块
   echo 4096 >  /sys/block/sda/queue/read_ahead_kb
   echo 4096 >  /sys/block/sdb/queue/read_ahead_kb
   写过期设置
   echo 1000 > /sys/block/sda/queue/iosched/write_expire
   echo 1000 > /sys/block/sdb/queue/iosched/write_expire
   读过期设置，写得1/2
   echo 500 > /sys/block/sda/queue/iosched/read_expire
   echo 500 > /sys/block/sdb/queue/iosched/read_expire
   linux I/O调度方式启用异步方式，提高读写性能
   echo 40 > /proc/sys/vm/dirty_ratio
   echo 1500 > /proc/sys/vm/dirty_expire_centisecs
   echo 1000 > /proc/sys/vm/dirty_writeback_centisecs
   echo 20 > /proc/sys/vm/dirty_background_ratio
8. LVM mount优化 （关闭记录文件系统atime）
   mount -o remount,noatime,nodiratime /dev/mapper/ubuntu--vg-nfs_lv 
9. 修改文件日志的提交时间
   mount –o remount,commit=15 DEVICE
10. nfs mount优化
11. 内核及网络优化

    net.core.netdev_max_backlog = 30000

    net.core.somaxconn = 262144

    net.core.rmem_default = 8388608

    net.core.wmem_default = 8388608

    net.core.rmem_max=16777216

    net.core.wmem_max=16777216

    net.ipv4.ip_local_port_range = 2048 65000

    net.ipv4.tcp_mem = 786432 1048576 1572864

    net.ipv4.tcp_rmem = 4096 87380 16777216

    net.ipv4.tcp_wmem = 4096 65536 16777216

    net.ipv4.tcp_fin_timeout = 10

    #net.ipv4.tcp_tw_recycle = 1

    net.ipv4.tcp_tw_reuse = 1

    net.ipv4.tcp_max_tw_buckets = 6000

    net.ipv4.tcp_timestamps = 0

    net.ipv4.tcp_window_scaling = 1

    net.ipv4.tcp_sack = 1

    net.ipv4.tcp_no_metrics_save=1

    net.ipv4.tcp_keepalive_time = 1200

    net.ipv4.tcp_syncookies = 1

    net.ipv4.tcp_max_orphans = 262144

    net.ipv4.tcp_max_syn_backlog = 262144

    net.ipv4.tcp_synack_retries = 2

    net.ipv4.tcp_syn_retries = 2

    #net.ipv4.netfilter.ip_conntrack_tcp_timeout_established = 1800

    #net.ipv4.netfilter.ip_conntrack_max = 131070

    kernel.sysrq = 0

    kernel.core_uses_pid = 1

    kernel.msgmnb = 65536

    kernel.msgmax = 65536

    kernel.shmmax = 68719476736

    kernel.shmall = 4294967296

    kernel.randomize_va_space = 1

    #kernel.exec-shield = 1

    net.ipv4.ip_forward = 0

    net.ipv4.neigh.default.gc_stale_time=120

    net.ipv4.conf.default.accept_source_route = 0

    net.ipv4.conf.default.rp_filter=0

    net.ipv4.conf.all.rp_filter=0

    net.ipv4.conf.all.arp_announce=2

    net.ipv4.conf.lo.arp_announce=2

    fs.file-max = 2000000

    fs.nr_open = 2000000

12. 打开net bbr算法
    # enable bbr

    net.core.default_qdisc = fq

    net.ipv4.tcp_congestion_control = bbr