安装
请点击下载hadoop-2.6.4
解压到 /home/hdfs/hadoop
1 useradd hdfs // 创建hdfs用户
2 passwd hdfs // 设置hdfs用户的登陆密码
3 su hdfs // 切换到hdfs用户登陆
4 cd ~ // 进入hdfs的家目录
5 rz // 上传windows本地的文件到此家目录
6 tar -zxvf hadoop-2.6.4.tar.gz // 解压安装包
7 mv hadoop-2.6.4 hadoop // 重命名解压后的文件名
hdfs机器免密验证设置
1 cd ~/.ssh
2 ssh-keygen -t rsa // 一路摁回车
3 id_rsa // 私钥
4 id_rsa.pub // 公钥
5 cat id_rsa.pub >> authorized_keys // 在master上将公钥放到authorized_keys(已认证的keys)里
6 sudo scp authorized_keys hdfs@192.168.153.129:~/.ssh
7 // 将master上的authorized_keys放到其他(需要免密的节点)linux的~/.ssh目录下
8 sudo scp authorized_keys hdfs@192.168.153.128:~/.ssh
9 // 将master上的authorized_keys放到其他(需要免密的节点)linux的~/.ssh目录下
10 chmod 644 authorized_keys // 修改钥的权限
设置Java环境变量
1 vi ~/bashrc
2 export JAVA_HOME=/java_install_dir/jdk1.8.0_91
3 export JRE_HOME=/java_install_dir/jdk1.8.0_91/jre
4 export CLASSPATH=.:
C
L
A
S
S
P
A
T
H
:
CLASSPATH:
CLASSPATH:JAVA_HOME/lib:
J
R
E
H
O
M
E
/
l
i
b
5
e
x
p
o
r
t
P
A
T
H
=
JRE_HOME/lib 5 export PATH=
JREHOME/lib5exportPATH=PATH:
J
A
V
A
H
O
M
E
/
b
i
n
:
JAVA_HOME/bin:
JAVAHOME/bin:JRE_HOME/bin
6 source ~/.bashrc // 让配置生效
7 echo $JAVA_HOME
8 /java_install_dir/jdk1.8.0_91
配置 *.xml
配置hadoop-env.sh
1 export JAVA_HOME=/java_install_dir/jdk1.8.0_91
配置 yarn-env.sh
1 export JAVA_HOME=/java_install_dir/jdk1.8.0_91
配置 core-site.xml
1
2
3 fs.defaultFS
4 hdfs://wtf-ubuntu:9000
5
6
7 io.file.buffer.size
8 131072
9
10
11 hadoop.tmp.dir
12 file:/home/hdfs/hadoop/tmp
13 Abasefor other temporary directories.
14
15
16 hadoop.proxyuser.spark.hosts
17
18
19
20 hadoop.proxyuser.spark.groups
21
22
配置hdfs-site.xml
dfs.namenode.secondary.http-address
wtf-ubuntu:9001
<property>
<name>dfs.namenode.name.dir</name>
<value>file:/home/hdfs/hadoop/name</value>
</property>
<property>
<name>dfs.datanode.data.dir</name>
<value>file:/home/hdfs/hadoop/data</value>
</property>
<property>
<name>dfs.replication</name>
<value>3</value>
</property>
<property>
<name>dfs.webhdfs.enabled</name>
<value>true</value>
</property>
配置 mapred-site.xml 文件
cp mapred-site.xml.template mapred-site.xml // 复制一份模板重命名为 mapred-site.xml
mapreduce.framework.name
yarn
mapreduce.jobhistory.address
wtf-ubuntu:10020
mapreduce.jobhistory.webapp.address
wtf-ubuntu:19888
配置 yarn-site.xml
yarn.nodemanager.aux-services
mapreduce_shuffle
yarn.nodemanager.aux-services.mapreduce.shuffle.class
org.apache.hadoop.mapred.ShuffleHandler
yarn.resourcemanager.address
wtf-ubuntu:8032
yarn.resourcemanager.scheduler.address
wtf-ubuntu:8030
yarn.resourcemanager.resource-tracker.address
wtf-ubuntu:8035
yarn.resourcemanager.admin.address
wtf-ubuntu:8033
yarn.resourcemanager.webapp.address
wtf-ubuntu:8088
启动
格式化NameNode
./bin/hdfs namenode -format
1
操作HDFS
./sbin/start-dfs.sh
./sbin/stop-dfs.sh
./sbin/start-yarn.sh
./sbin/stop-yarn.sh
./bin/hdfs dfsadmin -report
./bin/hadoop fs -mkdir /tmp
./bin/hadoop fs -mkdir /tmp/input
./bin/hadoop fs -ls /tmp/
./bin/hadoop fs -put logs/ /tmp
./bin/hadoop fs -ls /tmp/
./bin/hadoop fs -cat /tmp/logs/
./bin/hadoop fs -cat /tmp/logs/hadoop-hdfs-namenode-wtf-ubuntu.log