HDFS

安装
请点击下载hadoop-2.6.4

解压到 /home/hdfs/hadoop

1 useradd hdfs // 创建hdfs用户
2 passwd hdfs // 设置hdfs用户的登陆密码
3 su hdfs // 切换到hdfs用户登陆
4 cd ~ // 进入hdfs的家目录
5 rz // 上传windows本地的文件到此家目录
6 tar -zxvf hadoop-2.6.4.tar.gz // 解压安装包
7 mv hadoop-2.6.4 hadoop // 重命名解压后的文件名

hdfs机器免密验证设置
1 cd ~/.ssh
2 ssh-keygen -t rsa // 一路摁回车
3 id_rsa // 私钥
4 id_rsa.pub // 公钥
5 cat id_rsa.pub >> authorized_keys // 在master上将公钥放到authorized_keys(已认证的keys)里
6 sudo scp authorized_keys hdfs@192.168.153.129:~/.ssh
7 // 将master上的authorized_keys放到其他(需要免密的节点)linux的~/.ssh目录下
8 sudo scp authorized_keys hdfs@192.168.153.128:~/.ssh
9 // 将master上的authorized_keys放到其他(需要免密的节点)linux的~/.ssh目录下
10 chmod 644 authorized_keys // 修改钥的权限

设置Java环境变量
1 vi ~/bashrc
2 export JAVA_HOME=/java_install_dir/jdk1.8.0_91
3 export JRE_HOME=/java_install_dir/jdk1.8.0_91/jre
4 export CLASSPATH=.:$CLASSPATH:$JAVA_HOME/lib:$JR{E}_{H}OME/lib5exportPATH=$PATH:$JAV{A}_{H}OME/bin:$JRE_HOME/bin
6 source ~/.bashrc // 让配置生效
7 echo $JAVA_HOME
8 /java_install_dir/jdk1.8.0_91

配置 *.xml
配置hadoop-env.sh
1 export JAVA_HOME=/java_install_dir/jdk1.8.0_91

配置 yarn-env.sh
1 export JAVA_HOME=/java_install_dir/jdk1.8.0_91

配置 core-site.xml
1
2
3 fs.defaultFS
4 hdfs://wtf-ubuntu:9000
5
6
7 io.file.buffer.size
8 131072
9
10
11 hadoop.tmp.dir
12 file:/home/hdfs/hadoop/tmp
13 Abasefor other temporary directories.
14
15
16 hadoop.proxyuser.spark.hosts
17
18
19
20 hadoop.proxyuser.spark.groups
21
22

配置hdfs-site.xml


dfs.namenode.secondary.http-address
wtf-ubuntu:9001

<property>
    <name>dfs.namenode.name.dir</name>
    <value>file:/home/hdfs/hadoop/name</value>
</property>

<property>
    <name>dfs.datanode.data.dir</name>
    <value>file:/home/hdfs/hadoop/data</value>
</property>

<property>
    <name>dfs.replication</name>
    <value>3</value>
</property>

<property>
    <name>dfs.webhdfs.enabled</name>
    <value>true</value>
</property>

配置 mapred-site.xml 文件
cp mapred-site.xml.template mapred-site.xml // 复制一份模板重命名为 mapred-site.xml


mapreduce.framework.name
yarn


mapreduce.jobhistory.address
wtf-ubuntu:10020


mapreduce.jobhistory.webapp.address
wtf-ubuntu:19888

配置 yarn-site.xml


yarn.nodemanager.aux-services
mapreduce_shuffle


yarn.nodemanager.aux-services.mapreduce.shuffle.class
org.apache.hadoop.mapred.ShuffleHandler


yarn.resourcemanager.address
wtf-ubuntu:8032


yarn.resourcemanager.scheduler.address
wtf-ubuntu:8030


yarn.resourcemanager.resource-tracker.address
wtf-ubuntu:8035


yarn.resourcemanager.admin.address
wtf-ubuntu:8033


yarn.resourcemanager.webapp.address
wtf-ubuntu:8088

启动
格式化NameNode
./bin/hdfs namenode -format
1
操作HDFS
./sbin/start-dfs.sh
./sbin/stop-dfs.sh
./sbin/start-yarn.sh
./sbin/stop-yarn.sh
./bin/hdfs dfsadmin -report

./bin/hadoop fs -mkdir /tmp
./bin/hadoop fs -mkdir /tmp/input
./bin/hadoop fs -ls /tmp/
./bin/hadoop fs -put logs/ /tmp
./bin/hadoop fs -ls /tmp/
./bin/hadoop fs -cat /tmp/logs/
./bin/hadoop fs -cat /tmp/logs/hadoop-hdfs-namenode-wtf-ubuntu.log

web端查看
http://192.168.153.130:50070/
1

转载自：https://blog.csdn.net/wtf0215/article/details/52039744