hadoop 配置和一些重要的linux命令

1.配置nodes无密钥连接
配置环境变量
vi /etc/profile

指定ip
vi /etc/hosts

vim /etc/sysconfig/network-script/ifcfg-eth0

指定mac地址
vim /etc/udev/rules.d/70-persistent-net.rules

单机免密登陆
ssh-keygen -t dsa -P ‘’ -f ~/.ssh/id_dsa
cat ~/.ssh/id_dsa.pub >> ~/.ssh/authorized_keys

上传工具
yum -y install lrzsz
rz -E

多机免密登陆
ssh-keygen -t rsa
ssh-copy-id hadoop1
scp /root/.ssh/authorized_keys hadoop2:/root/.ssh

时钟同步
yum install -y ntp
crontab -e
*/1 * * * * /usr/sbin/ntpdate ntp4.aliyun.com;

配置路由表
vim /etc/sysconfig/network-scripts/ifcfg-配置_1
service network restart

指定hostname
vi /etc/hostname

rpm -qa | grep jdk 查看java
yum -y remove
查看jdk路径
echo $JAVA_HOME

修改文件名
cp 原文件 改好的

2.关闭防火墙
service iptables stop
chkconfig iptables off

关闭selinux
vi /etc/selinux/config

3.修改/etc/hosts

4.解压Hadoop tar文件
cd hadoop-2.2.0
mkdir tmp

5.配置hadoop/etc下面的文件
cd etc/hadoop

1)配置hadoop-env.sh
export JAVA_HOME=/usr/java/jdk1.6.0_35

2)配置yarn-env.sh
export JAVA_HOME=/usr/java/jdk1.6.0_35

3)配置slaves
n1
n2

4)配置core-site.xml

fs.defaultFS
hdfs://h1:9000

    <property>
            <name>hadoop.tmp.dir</name>
            <value>file:/root/hadoop-2.2.0/tmp</value>
    </property>

5)配置hdfs-site.xml

dfs.namenode.secondary.http-address
h1:9001

<property>
    	<name>dfs.namenode.name.dir</name>
    	<value>file:/root/hadoop-2.2.0/tmp/dfs/name</value>
</property>

<property>
    	<name>dfs.datanode.data.dir</name>
    	<value>file:/root/hadoop-2.2.0/tmp/dfs/data</value>
</property>

    <property>
            <name>dfs.replication</name>
            <value>1</value>
    </property>

6)配置mapred-site.xml
cp mapred-site.xml.template mapred-site.xml

    <property>
            <name>mapreduce.framework.name</name>
            <value>yarn</value>
    </property>

7)配置yarn-site.xml

yarn.resourcemanager.address
h1:8032

<property>
	<name>yarn.resourcemanager.scheduler.address</name>
	<value>h1:8030</value>
</property>

<property>
	<name>yarn.resourcemanager.resource-tracker.address</name>
	<value>h1:8031</value>
</property>

<property>
	<name>yarn.resourcemanager.admin.address</name>
	<value>h1:8033</value>
</property>

    <property>
            <name>yarn.nodemanager.aux-services</name>
            <value>mapreduce_shuffle</value>
    </property>
    <property>
            <name>yarn.nodemanager.aux-services.mapreduce.shuffle.class</name>
            <value>org.apache.hadoop.mapred.ShuffleHandler</value>
    </property>

6.编写复制hadoop到slaves的脚本cp2slave.sh
#!/bin/bash

scp -r /root/hadoop-2.2.0 root@n1:~/
scp -r /root/hadoop-2.2.0 root@n2:~/

7.初始化namenode
hdfs namenode -format

8.运行hadoop

hadoop jar share/hadoop/mapreduce/hadoop-mapreduce-examples-2.2.0.jar wordcount /input /output