hadoop集群优化

hadoop集群优化：
一、配置参数优化
1、小文件优化：调整split\reduce等参数
split参数需根据应用自定义调整,不能统一配置。
--hive-site.xml（调整reduce输出参数）
<property>
    <name>hive.merge.mapredfiles</name>
    <value>true</value>
</property>
<property>
    <name>hive.merge.smallfiles.avgsize</name>
    <value>100000000</value>
</property>

操作步骤：
    ---检查各节点hive-site.xml配置是否一致
    ./drun "wc -l /home/hadoop/hive/conf/hive-site.xml"
    检查结果不一致,查看配置参数,部分节点新增了一些参数，基于hive特性，可在82节点配齐所有参数后同步
    
    批量更新从节点
    ./drun "cp /home/hadoop/hive/conf/hive-site.xml /home/hadoop/hive/conf/hive-site.xml_`date +%Y%m%d`_bak1"
    ./dcopy -r /home/hadoop/hive/conf/hive-site.xml /home/hadoop/hive/conf/
    单独更新standby节点
    cp /home/hadoop/hive/conf/hive-site.xml /home/hadoop/hive/conf/hive-site.xml_`date +%Y%m%d`_bak1
    scp /home/hadoop/hive/conf/hive-site.xml bis-backup-s-01:/home/hadoop/hive/conf/
2、内存参数(目前map/1g,reduce/3g)是否合适？ NM总分配内存12G(原则上分配总内存（32G）的70-80%),可否调大？(目前上面还承载着spark/hbase)
yarn.nodemanager.resource.cpu-vcores （采用了默认值8核，但总核数12核建议调大）
节点内存不统一，按如下分组配置。
7台12core 32g配置
    os 预留 10% 分配4g
    yarn 10c/15360m
    hbase 6g
    spark  worker 4g/4c
    该配置核数受限，会出现cpu抢占情况。
20台 24core 16g配置
    spark  worker 4g/4c
    yarn   8c/12g
    由于本身硬件考虑不周到（核数比内存还大），故暂不给OS预留，峰值时期会是瓶颈。同时core本身会浪费
19台 24core 32g配置
    os 预留 10% 分配4g
    yarn 12c/18432m
    hbase 6g
    spark  worker 4g/4c
(mapreduce.map.memory.mb,mapreduce.map.cpu.vcores)=(1500M,1)
(mapreduce.reduce.memory.mb,mapreduce.reduce.cpu.vcores)=(3000M,1)
a、针对32g/12c节点
--yarn-site.xml
<property>
    <name>yarn.nodemanager.resource.cpu-vcores</name>
    <value>10</value>
</property>

<property>
    <name>yarn.nodemanager.resource.memory-mb</name>
    <value>15360</value>
</property>

<!--开启日志汇聚-->
<property>
    <name>yarn.log-aggregation-enable</name>
    <value>true</value>
</property>





b、针对16g节点<不调整,采用原先值>

c、针对24core 32g配置
<property>
    <name>yarn.nodemanager.resource.cpu-vcores</name>
    <value>12</value>
</property>

<property>
    <name>yarn.nodemanager.resource.memory-mb</name>
    <value>18432</value>
</property>

<!--开启日志汇聚-->
<property>
    <name>yarn.log-aggregation-enable</name>
    <value>true</value>
</property>

操作步骤：
    ---检查各节点yarn-site.xml配置是否一致
./drun "wc -l /home/hadoop/hadoop-2.2.0/etc/hadoop/yarn-site.xml"
配置一致
  ---批量更新32g从节点

./drun "cp /home/hadoop/hadoop-2.2.0/etc/hadoop/yarn-site.xml /home/hadoop/hadoop-2.2.0/etc/hadoop/yarn-site.xml_`date +%Y%m%d`_bak1"
./dcopy -r /home/hadoop/hadoop-2.2.0/etc/hadoop/yarn-site.xml /home/hadoop/hadoop-2.2.0/etc/hadoop/
单独更新standby节点
    cp /home/hadoop/hadoop-2.2.0/etc/hadoop/yarn-site.xml /home/hadoop/hadoop-2.2.0/etc/hadoop/yarn-site.xml_`date +%Y%m%d`_bak1
    scp /home/hadoop/hadoop-2.2.0/etc/hadoop/yarn-site.xml bis-backup-s-01:/home/hadoop/hadoop-2.2.0/etc/hadoop/

--停hbase服务
yarn rmadmin -refreshNodes
3、开启默认压缩功能，推荐 lzo or snappy 压缩
本次暂时不处理，要在各节点把lzo客户端装上，由应用程序决定是否压缩
二、内核优化
1、调整透明大页
if test -f /sys/kernel/mm/transparent_hugepage/enabled; then
   echo never > /sys/kernel/mm/redhat_transparent_hugepage/enabled
fi
if test -f /sys/kernel/mm/redhat_transparent_hugepage/defrag; then
   echo never > /sys/kernel/mm/redhat_transparent_hugepage/defrag
fi

chmod +x /etc/rc.d/rc.local

-----10.10.10.159做了root互信(补充了7台机器root互信),在此机器上同步

./drun "cp /etc/rc.d/rc.local /etc/rc.d/rc.local_`date +%Y%m%d`_bak1"
将以上内容增加到 transparenthugesize.txt中
./dcopy -r ~/transparenthugesize.txt ~/
./drun "cat ~/transparenthugesize.txt >>/etc/rc.d/rc.local"
./drun "chmod +x /etc/rc.d/rc.local"
./drun "echo never > /sys/kernel/mm/redhat_transparent_hugepage/enabled"
./drun "echo never > /sys/kernel/mm/redhat_transparent_hugepage/defrag"


2、调整swap参数
/etc/sysctl.conf
    vm.swappiness=10
sysctl vm.swappiness=10

echo vm.swappiness=10 >swappiness.txt
./drun "cp /etc/sysctl.conf /etc/sysctl.conf_`date +%Y%m%d`_bak1"
./dcopy -r ~/swappiness.txt ~/
./drun "cat ~/swappiness.txt >>/etc/sysctl.conf"
./drun "sysctl vm.swappiness=10"
./drun "cat /proc/sys/vm/swappiness"

三、运维监控优化(参数已在第一步调整)
1、rm监控界面部分Job无法查阅历史日志
./mr-jobhistory-daemon.sh start historyserver
启用mr-history-server后正常,后面再拿应用rpt_stl_img测试
2、开启日志汇聚功能<之前即已打开>
--yarn-site.xml
set yarn.log-aggregation-enable=true;
set yarn.log-aggregation.retain-seconds=864000;
set yarn.log-aggregation.retain-check-interval-seconds=-1 --默认值
set yarn.nodemanager.remote-app-log-dir=/tmp/hadoop-yarn/staging/contains;
hadoop fs -mkdir /tmp/hadoop-yarn/staging/contains
hadoop fs -chmod 777 /tmp/hadoop-yarn/staging/contains
hdfs dfsadmin -refreshNodes
yarn rmadmin -refreshNodes
<!--开启日志汇聚-->
<property>
    <name>yarn.log-aggregation-enable</name>
    <value>true</value>
</property>

四、配置 lzo
1、maven准备
wget http://mirrors.shuosc.org/apache/maven/maven-3/3.5.0/binaries/apache-maven-3.5.0-bin.tar.gz
tar -zxvf apache-maven-3.5.0-bin.tar.gz -C ../
mv apache-maven-3.5.0 maven

修改maven 配置
vi /etc/profile
export M2=/root/maven
export PATH=${PATH}:$M2/bin
source /etc/profile
检查maven是否安装成功
mvn -version
conf/settings.xml中添加国内镜像配置
<mirror>
  <id>alimaven</id>
  <name>aliyun maven</name>
  <url>http://maven.aliyun.com/nexus/content/groups/public/</url>
  <mirrorOf>central</mirrorOf>
</mirror>
<mirror>
  <id>repo2</id>
  <mirrorOf>central</mirrorOf>
  <name>Human Readable Name for this Mirror.</name>
  <url>http://repo2.maven.org/maven2/</url>
</mirror>
<mirror>
  <id>ibiblio</id>
  <mirrorOf>central</mirrorOf>
  <name>ibiblio Mirror of http://repo1.maven.org/maven2/</name>
  <url>http://mirrors.ibiblio.org/pub/mirrors/maven2/</url>
</mirror>
<mirror>
    <id>jboss-public-repository-group</id>
    <mirrorOf>central</mirrorOf>
    <name>JBoss Public Repository Group</name>
    <url>http://repository.jboss.org/nexus/content/groups/public</url>
</mirror>


2、安装lzo（之前已在测试集群安装过，故直接拷贝即可）
./dcopy -r /root/tmpdir/lzo/lib/\* /usr/lib64

---下面测试集群详细部署步骤
wget http://www.oberhumer.com/opensource/lzo/download/lzo-2.06.tar.gz
tar -zxvf lzo-2.06.tar.gz -C ../
cd lzo-2.06
mkdir -p /usr/local/lzo206
./configure --enable-shared --prefix=/usr/local/lzo206
make && make install
将/usr/local/lzo206 目录下的所有文件打包，并同步到集群中的所有机器上。
scp bis-newdatanode-s2b-80:/usr/local/lzo206/lib /root/tmpdir/lzo
./dcopy -r /root/tmpdir/lzo/lib/\* /usr/lib64
3、设置环境变量
vi /etc/profile
export C_INCLUDE_PATH=/usr/local/lzo206/include
export LIBRARY_PATH=/usr/local/lzo206/lib

3、安装hadoop-lzo
之前已在111测试集群安装过，故直接拷贝即可
先拷贝到80集群主节点--hadoop账号
scp bis-newdatanode-s2b-80:/home/hadoop/hadoop-2.2.0/share/hadoop/common/hadoop-lzo-0.4.21-SNAPSHOT.jar /home/hadoop/hadoop-2.2.0/share/hadoop/common/
scp bis-newdatanode-s2b-80:/home/hadoop/hadoop-2.2.0/lib/native/libgplcompression* /home/hadoop/hadoop-2.2.0/lib/native/

scp /home/hadoop/hadoop-2.2.0/share/hadoop/common/hadoop-lzo-0.4.21-SNAPSHOT.jar bis-backup-s-01:/home/hadoop/hadoop-2.2.0/share/hadoop/common/
scp /home/hadoop/hadoop-2.2.0/lib/native/libgplcompression* bis-backup-s-01:/home/hadoop/hadoop-2.2.0/lib/native/

分发到各从节点
./dcopy -r /home/hadoop/hadoop-2.2.0/share/hadoop/common/hadoop-lzo-0.4.21-SNAPSHOT.jar /home/hadoop/hadoop-2.2.0/share/hadoop/common/
./dcopy -r /home/hadoop/hadoop-2.2.0/lib/native/libgplcompression\* /home/hadoop/hadoop-2.2.0/lib/native/


修改spark-env.sh,增加spark 读lzo文件配置：
export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/home/hadoop/hadoop-2.2.0/lib/native
export SPARK_LIBRARY_PATH=$SPARK_LIBRARY_PATH:/home/hadoop/hadoop-2.2.0/lib/native
export SPARK_CLASSPATH=$SPARK_CLASSPATH:/home/hadoop/hadoop-2.2.0/share/hadoop/yarn/*:/home/hadoop/hadoop-2.2.0/share/hadoop/yarn/lib/*:/home/hadoop/hadoop-2.2.0/share/hadoop/common/*:/home/hadoop/hadoop-2.2.0/share/hadoop/common/lib/*:/home/hadoop/hadoop-2.2.0/share/hadoop/hdfs/*:/home/hadoop/hadoop-2.2.0/share/hadoop/hdfs/lib/*:/home/hadoop/hadoop-2.2.0/share/hadoop/mapreduce/*:/home/hadoop/hadoop-2.2.0/share/hadoop/mapreduce/lib/*:/home/hadoop/hadoop-2.2.0/share/hadoop/tools/lib/*:/home/hadoop/spark-1.6.1-bin-2.2.0/lib/*

./dcopy -r /home/hadoop/spark-1.6.1-bin-2.2.0/conf/spark-env.sh /home/hadoop/spark-1.6.1-bin-2.2.0/conf/


---下面测试集群详细部署步骤
 wget https://github.com/twitter/hadoop-lzo/archive/master.zip
 unzip master.zip -d ../
 cd hadoop-lzo-master
 修改pom.xml中的hadoop.current.version值为当前版本2.2.0

 mvn clean package -Dmaven.test.skip=true
 
 cp target/hadoop-lzo-0.4.21-SNAPSHOT.jar /home/hadoop/hadoop-2.2.0/share/hadoop/common
 cp /root/hadoop-lzo-master/target/native/Linux-amd64-64/lib/* /usr/lib64
 cp /root/hadoop-lzo-master/target/native/Linux-amd64-64/lib/* /home/hadoop/hadoop-2.2.0/lib/native
 chown -R hadoop.hadoop /home/hadoop/hadoop-2.2.0/share/hadoop/common/* /home/hadoop/hadoop-2.2.0/lib/native/*
 
 分发至各个节点
 
 ./dcopy -r /home/hadoop/hadoop-2.2.0/share/hadoop/common/hadoop-lzo-0.4.21-SNAPSHOT.jar /home/hadoop/hadoop-2.2.0/share/hadoop/common/
 ./dcopy -r /home/hadoop/hadoop-2.2.0/lib/native/libgplcompression\* /home/hadoop/hadoop-2.2.0/lib/native/



配置 core-site.xml
<!--add lzo compress-->
<property>  
　　<name>io.compression.codecs</name>  
    <value>
          org.apache.hadoop.io.compress.GzipCodec,org.apache.hadoop.io.compress.DefaultCodec,  
          com.hadoop.compression.lzo.LzoCodec,com.hadoop.compression.lzo.LzopCodec,org.apache.hadoop.io.compress.BZip2Codec,org.apache.hadoop.io.compress.SnappyCodec
    </value>  
</property>  
 
<property>  
　　<name>io.compression.codec.lzo.class</name>  
　　<value>com.hadoop.compression.lzo.LzoCodec</value>  
</property>

./drun "ls -l /home/hadoop/hadoop-2.2.0/etc/hadoop/core-site.xml*"
mv /home/hadoop/hadoop-2.2.0/etc/hadoop/core-site.xml_20171020_bak2 /home/hadoop/hadoop-2.2.0/etc/hadoop/core-site.xml
./drun "mv /home/hadoop/hadoop-2.2.0/etc/hadoop/core-site.xml_20171020_bak2 /home/hadoop/hadoop-2.2.0/etc/hadoop/core-site.xml"

./drun "cp /home/hadoop/hadoop-2.2.0/etc/hadoop/core-site.xml /home/hadoop/hadoop-2.2.0/etc/hadoop/core-site.xml_`date +%Y%m%d`_bak1"
./dcopy -r /home/hadoop/hadoop-2.2.0/etc/hadoop/core-site.xml /home/hadoop/hadoop-2.2.0/etc/hadoop/

scp /home/hadoop/hadoop-2.2.0/etc/hadoop/core-site.xml bis-backup-s-01:/home/hadoop/hadoop-2.2.0/etc/hadoop/

重启服务
hadoop重启
master --10.10.10.82

/home/hadoop/hadoop-2.2.0/sbin/stop-all.sh
/home/hadoop/hadoop-2.2.0/sbin/start-all.sh
mr-jobhistory-daemon.sh start historyserver
namenode切换活跃状态
bis-hadoop-namenode-s-01
bis-backup-s-01
hdfs haadmin -getServiceState bis-hadoop-namenode-s-01
hdfs haadmin -getServiceState bis-backup-s-01
hdfs haadmin -transitionToActive --forcemanual bis-hadoop-namenode-s-01

/home/hadoop/hadoop-2.2.0/sbin/hadoop-daemon.sh start namenode
/home/hadoop/hadoop-2.2.0/sbin/hadoop-daemon.sh start datanode
/home/hadoop/hadoop-2.2.0/sbin/hadoop-daemon.sh start journalnode
/home/hadoop/hadoop-2.2.0/sbin/yarn-daemon.sh start nodemanager
/home/hadoop/hadoop-2.2.0/sbin/yarn-daemon.sh start resourcemanager
hive重启
hive --service hiveserver2 --hiveconf hive.server2.thrift.port=10000 > /home/hadoop/hive/hiveserver2.log 2>&1 &

bis-hadoop-datanode-s-02 节点：
nohup /home/hadoop/hive/bin/hive --service hiveserver2 > /home/hadoop/hive/hiveserver2.log 2>&1 &
下面3个节点启用hiveserver服务:
bis-hadoop-namenode-s-01
bis-hadoop-datanode-s-03
bis-hadoop-datanode-s-04
bis-hadoop-datanode-s2-05
nohup /home/hadoop/hive/bin/hive --service hiveserver > /home/hadoop/hive/hiveserver.log 2>&1 &

hbase重启
master --10.10.10.82
/home/hadoop/hbase-0.96.2-hadoop2/bin/stop-hbase.sh
/home/hadoop/hbase-0.96.2-hadoop2/bin/start-hbase.sh

/home/hadoop/hbase-0.96.2-hadoop2/bin/hbase-daemon.sh start regionserver
spark server重启 --10.10.10.80
sbin/start-thriftserver.sh --master spark://10.10.10.82:7077 --driver-memory 10G --total-executor-cores 160 --executor-memory 1g --executor-cores 1 --hiveconf hive.server2.thrift.port=10050 --conf spark.dynamicAllocation.enabled=false

master --10.10.10.82
/home/hadoop/spark-1.6.1-bin-2.2.0/sbin/stop-all.sh
/home/hadoop/spark-1.6.1-bin-2.2.0/sbin/start-all.sh


验证：
---load lzo文件 to hive table
create table test_lzo2(
id int,
name string)
STORED AS INPUTFORMAT 'com.hadoop.mapred.DeprecatedLzoTextInputFormat'
OUTPUTFORMAT 'org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat';

echo -e "2\001xjp">>11.txt
lzop 11.txt
load data local inpath '/home/hadoop/11.txt.lzo' overwrite into table test_lzo2;

--hive operator with lzo compression
set hive.exec.compress.output=true;
set mapred.output.compression.codec=com.hadoop.compression.lzo.LzoCodec;
set mapred.output.compression.type=BLOCK;
set io.seqfile.compression.type=BLOCK;

create table test_lzo8(
id int,
name string)
stored as sequencefile;

insert overwrite table test_lzo8 select * from test_lzo;



create table test_lzo6(
id int,
name string)
insert into test_lzo6 select * from test_lzo;




-----hbase
./drun "/home/hadoop/hbase-0.96.2-hadoop2/bin/hbase-daemon.sh start regionserver"
./drun "sed -i 's/^/#/g' /var/spool/cron/hadoop"