免登陆
ssh-keygen -t rsa -P ''
ssh-copy-id -i ~/.ssh/id_rsa.pub xxx.xxx.xxx.xxx
为Impala用途
mkdir /var/run/hadoop-hdfs
chmod -R g+w /var/run/hadoop-hdfs
相关目录权限:
sed -i "s/admin:x:600:/admin:x:600:hadp/g" /etc/group
chmod -R g+w /export
mkdir /export/hdfs
mkdir /export/hadoop-pids
mkdir -p /export/logs/hadoop
mkdir /export/logs/yarn
chown -R hadp:hadp /export/hdfs
chown -R hadp:hadp /export/logs/hadoop
chown -R hadp:hadp /export/logs/yarn
chown -R hadp:hadp /export/hadoop-pids
环境变量:
su - hadp
sed -i '$a \export HADOOP_HOME=/software/servers/hadoop-2.0.0-cdh4.7.0' ~/.bashrc
sed -i '$a \export HADOOP_CONF_DIR=$HADOOP_HOME/etc/hadoop/' ~/.bashrc
sed -i '$a \export HDFS_CONF_DIR=$HADOOP_HOME/etc/hadoop/' ~/.bashrc
sed -i '$a \export YARN_CONF_DIR=$HADOOP_HOME/etc/hadoop/' ~/.bashrc
sed -i '$a \export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/usr/local/lib:$HADOOP_HOME/lib/native' ~/.bashrc
sed -i '$a \export PATH=$PATH:$HADOOP_HOME/bin:$HADOOP_HOME/sbin' ~/.bashrc
source ~/.bashrc
HADOOP2 HA的配置
HADOOP2的HA有两种方式实现元数据共享,一种是操作系统级NFS,另一种是QJM。
NFS方式没有fencing功能,如果不设置有效的fencing方法,会导致“脑裂”问题。
所以QJM方式配置:
两个NAMENODE:
<property>
<name>dfs.federation.nameservices</name>
<value>cmo</value>
</property>
<property>
<name>dfs.namenode.name.dir</name>
<value>/export/hdfs/nn</value>
</property>
<property>
<name>dfs.ha.namenodes.cmo</name>
<value>nn1,nn2</value>
</property>
<property>
<name>dfs.namenode.rpc-address.cmo.nn1</name>
<value>A01-R06-I148-21.jd.local:54310</value>
</property>
<property>
<name>dfs.namenode.rpc-address.cmo.nn2</name>
<value>A01-R06-I148-22.jd.local:54310</value>
</property>
<property>
<name>dfs.namenode.http-address.cmo.nn1</name>
<value>A01-R06-I148-21.jd.local:50070</value>
</property>
<property>
<name>dfs.namenode.http-address.cmo.nn2</name>
<value>A01-R06-I148-22.jd.local:50070</value>
</property>
journal配置:
<property>
<name>dfs.journalnode.edits.dir</name>
<value>/export/hdfs/journal</value>
</property>
<property>
<name>dfs.namenode.shared.edits.dir</name>
<value>qjournal://A01-R06-I148-21.jd.local:8485;A01-R06-I148-22.jd.local:8485;A01-R06-I148-23.jd.local:8485/cmo</value>
</property>
<property>
<name>dfs.client.failover.proxy.provider.cmo</name>
<value>org.apache.hadoop.hdfs.server.namenode.ha.ConfiguredFailoverProxyProvider</value>
</property>
<property>
<name>dfs.ha.fencing.methods</name>
<value>sshfence</value>
</property>
<property>
<name>dfs.ha.fencing.ssh.private-key-files</name>
<value>/home/hadp/.ssh/id_rsa</value>
</property>
<property>
<name>dfs.ha.automatic-failover.enabled</name>
<value>true</value>
</property>
<property>
<name>ha.zookeeper.quorum</name>
<value>A01-R06-I148-18.jd.local,A01-R06-I148-19.jd.local,A01-R06-I148-20.jd.local</value>
</property>
<property>
<name>ha.zookeeper.parent-znode</name>
<value>/hadoop-ha-21-30</value>
</property>
启动方式:
hdfs zkfc -formatZK
启动三个journalnode
hadoop-daemon.sh start journalnode
格式化:
hadoop namenode -format
启动Namenode1:
hadoop-daemon.sh start namenode
同步Namenode2:
hdfs namenode -bootstrapStandby
hadoop-daemon.sh start namenode
hdfs haadmin -transitionToActive --forcemanual nn1
启动zkfc:
hadoop-daemon.sh start zkfc
启动datanode:
ssh-keygen -t rsa -P ''
ssh-copy-id -i ~/.ssh/id_rsa.pub xxx.xxx.xxx.xxx
为Impala用途
mkdir /var/run/hadoop-hdfs
chmod -R g+w /var/run/hadoop-hdfs
相关目录权限:
sed -i "s/admin:x:600:/admin:x:600:hadp/g" /etc/group
chmod -R g+w /export
mkdir /export/hdfs
mkdir /export/hadoop-pids
mkdir -p /export/logs/hadoop
mkdir /export/logs/yarn
chown -R hadp:hadp /export/hdfs
chown -R hadp:hadp /export/logs/hadoop
chown -R hadp:hadp /export/logs/yarn
chown -R hadp:hadp /export/hadoop-pids
环境变量:
su - hadp
sed -i '$a \export HADOOP_HOME=/software/servers/hadoop-2.0.0-cdh4.7.0' ~/.bashrc
sed -i '$a \export HADOOP_CONF_DIR=$HADOOP_HOME/etc/hadoop/' ~/.bashrc
sed -i '$a \export HDFS_CONF_DIR=$HADOOP_HOME/etc/hadoop/' ~/.bashrc
sed -i '$a \export YARN_CONF_DIR=$HADOOP_HOME/etc/hadoop/' ~/.bashrc
sed -i '$a \export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/usr/local/lib:$HADOOP_HOME/lib/native' ~/.bashrc
sed -i '$a \export PATH=$PATH:$HADOOP_HOME/bin:$HADOOP_HOME/sbin' ~/.bashrc
source ~/.bashrc
HADOOP2 HA的配置
HADOOP2的HA有两种方式实现元数据共享,一种是操作系统级NFS,另一种是QJM。
NFS方式没有fencing功能,如果不设置有效的fencing方法,会导致“脑裂”问题。
所以QJM方式配置:
两个NAMENODE:
<property>
<name>dfs.federation.nameservices</name>
<value>cmo</value>
</property>
<property>
<name>dfs.namenode.name.dir</name>
<value>/export/hdfs/nn</value>
</property>
<property>
<name>dfs.ha.namenodes.cmo</name>
<value>nn1,nn2</value>
</property>
<property>
<name>dfs.namenode.rpc-address.cmo.nn1</name>
<value>A01-R06-I148-21.jd.local:54310</value>
</property>
<property>
<name>dfs.namenode.rpc-address.cmo.nn2</name>
<value>A01-R06-I148-22.jd.local:54310</value>
</property>
<property>
<name>dfs.namenode.http-address.cmo.nn1</name>
<value>A01-R06-I148-21.jd.local:50070</value>
</property>
<property>
<name>dfs.namenode.http-address.cmo.nn2</name>
<value>A01-R06-I148-22.jd.local:50070</value>
</property>
journal配置:
<property>
<name>dfs.journalnode.edits.dir</name>
<value>/export/hdfs/journal</value>
</property>
<property>
<name>dfs.namenode.shared.edits.dir</name>
<value>qjournal://A01-R06-I148-21.jd.local:8485;A01-R06-I148-22.jd.local:8485;A01-R06-I148-23.jd.local:8485/cmo</value>
</property>
<property>
<name>dfs.client.failover.proxy.provider.cmo</name>
<value>org.apache.hadoop.hdfs.server.namenode.ha.ConfiguredFailoverProxyProvider</value>
</property>
<property>
<name>dfs.ha.fencing.methods</name>
<value>sshfence</value>
</property>
<property>
<name>dfs.ha.fencing.ssh.private-key-files</name>
<value>/home/hadp/.ssh/id_rsa</value>
</property>
<property>
<name>dfs.ha.automatic-failover.enabled</name>
<value>true</value>
</property>
<property>
<name>ha.zookeeper.quorum</name>
<value>A01-R06-I148-18.jd.local,A01-R06-I148-19.jd.local,A01-R06-I148-20.jd.local</value>
</property>
<property>
<name>ha.zookeeper.parent-znode</name>
<value>/hadoop-ha-21-30</value>
</property>
启动方式:
hdfs zkfc -formatZK
启动三个journalnode
hadoop-daemon.sh start journalnode
格式化:
hadoop namenode -format
启动Namenode1:
hadoop-daemon.sh start namenode
同步Namenode2:
hdfs namenode -bootstrapStandby
hadoop-daemon.sh start namenode
hdfs haadmin -transitionToActive --forcemanual nn1
启动zkfc:
hadoop-daemon.sh start zkfc
启动datanode:
hadoop-daemons.sh start datanode
no-ha迁移:
hdfs namenode -initializeSharedEdits