目录
1、hadoop调优实战
(1)调整namenode启动的堆内存大小
cd /hadoop-2.6.0-cdh5.14.2/etc/hadoop
vim hadoop-env.sh
export HADOOP_HEAPSIZE=8192m
export HADOOP_NAMENODE_INIT_HEAPSIZE="8192"
(2)调整jobHisotryServer的堆内存大小
cd /hadoop-2.6.0-cdh5.14.2/etc/hadoop
vim mapred-env.sh
export HADOOP_JOB_HISTORYSERVER_HEAPSIZE=4096
(3)调整yarn的堆内存大小
cd /hadoop-2.6.0-cdh5.14.2/etc/hadoop
vim yarn-env.sh
JAVA_HEAP_MAX=-Xmx8192m
export YARN_RESOURCEMANAGER_OPTS=-Xmx8192m -Xms8192m -XX:SurvivorRatio=8
(4)调整yarn内存大小
cd //hadoop-2.6.0-cdh5.14.2/etc/hadoop
vim yarn-site.xml
4096 | 每个container最小内存 | |
16384 | 每个container最大分配内存 | |
2 | 每个container最少分配CPU核数 | |
8 | 每个container最大CPU核数 | |
61440 | 每台机器的物理内存 | |
18 | 每台机器的CPU核数 |
2、flume组件内存调优
cd /apache-flume-1.6.0-cdh5.14.2-bin/conf
vim flume-env.sh
export JAVA_OPTS="-Xms4096m –Xmx4096m -Dcom.sun.management.jmxremote"
export JAVA_OPTS="$JAVA_OPTS -Dorg.apache.flume.log.rawdata=true -Dorg.apache.flume.log.printconfig=true "
3、Hbase内存调优
cd /hbase-1.2.0-cdh5.14.2/conf
vim hbase-env.sh
export HBASE_HEAPSIZE=8G
export HBASE_OFFHEAPSIZE=16G
export HBASE_MASTER_OPTS="$HBASE_MASTER_OPTS -XX:PermSize=16384m -XX:MaxPermSize=16384m -XX:ReservedCodeCacheSize=2048m"
export SERVER_GC_OPTS="-verbose:gc -XX:+PrintGCDetails -XX:+PrintGCDateStamps"
export SERVER_GC_OPTS="-verbose:gc -XX:+PrintGCDetails -XX:+PrintGCDateStamps -Xloggc:/kkb/install/gc/gc.log -XX:+UseGCLogFileRotation -XX:Numb
erOfGCLogFiles=1 -XX:GCLogFileSize=512M"