HDFS 单机搭建
标签 : Java hadoop hdfs
安装
请点击下载hadoop-2.6.4
解压到
/home/hdfs/hadoop
useradd hdfs // 创建hdfs用户
passwd hdfs // 设置hdfs用户的登陆密码
su hdfs // 切换到hdfs用户登陆
cd ~ // 进入hdfs的家目录
rz // 上传windows本地的文件到此家目录
tar -zxvf hadoop-2.6.4.tar.gz // 解压安装包
mv hadoop-2.6.4 hadoop // 重命名解压后的文件名
- hdfs机器免密验证设置
cd ~/.ssh
ssh-keygen -t rsa // 一路摁回车
id_rsa // 私钥
id_rsa.pub // 公钥
cat id_rsa.pub >> authorized_keys // 在master上将公钥放到authorized_keys(已认证的keys)里
sudo scp authorized_keys hdfs@192.168.153.129:~/.ssh
// 将master上的authorized_keys放到其他(需要免密的节点)linux的~/.ssh目录下
sudo scp authorized_keys hdfs@192.168.153.128:~/.ssh
// 将master上的authorized_keys放到其他(需要免密的节点)linux的~/.ssh目录下
chmod 644 authorized_keys // 修改钥的权限
- 设置Java环境变量
vi ~/bashrc
export JAVA_HOME=/java_install_dir/jdk1.8.0_91
export JRE_HOME=/java_install_dir/jdk1.8.0_91/jre
export CLASSPATH=.:$CLASSPATH:$JAVA_HOME/lib:$JRE_HOME/lib
export PATH=$PATH:$JAVA_HOME/bin:$JRE_HOME/bin
source ~/.bashrc // 让配置生效
echo $JAVA_HOME
/java_install_dir/jdk1.8.0_91
配置 *.xml
- 配置hadoop-env.sh
export JAVA_HOME=/java_install_dir/jdk1.8.0_91
- 配置 yarn-env.sh
export JAVA_HOME=/java_install_dir/jdk1.8.0_91
- 配置 core-site.xml
<configuration>
<property>
<name>fs.defaultFS</name>
<value>hdfs://wtf-ubuntu:9000</value>
</property>
<property>
<name>io.file.buffer.size</name>
<value>131072</value>
</property>
<property>
<name>hadoop.tmp.dir</name>
<value>file:/home/hdfs/hadoop/tmp</value>
<description>Abasefor other temporary directories.</description>
</property>
<property>
<name>hadoop.proxyuser.spark.hosts</name>
<value>*</value>
</property>
<property>
<name>hadoop.proxyuser.spark.groups</name>
<value>*</value>
</property>
</configuration>
- 配置hdfs-site.xml
<configuration>
<property>
<name>dfs.namenode.secondary.http-address</name>
<value>wtf-ubuntu:9001</value>
</property>
<property>
<name>dfs.namenode.name.dir</name>
<value>file:/home/hdfs/hadoop/name</value>
</property>
<property>
<name>dfs.datanode.data.dir</name>
<value>file:/home/hdfs/hadoop/data</value>
</property>
<property>
<name>dfs.replication</name>
<value>3</value>
</property>
<property>
<name>dfs.webhdfs.enabled</name>
<value>true</value>
</property>
</configuration>
- 配置 mapred-site.xml 文件
cp mapred-site.xml.template mapred-site.xml // 复制一份模板重命名为 mapred-site.xml
<configuration>
<property>
<name>mapreduce.framework.name</name>
<value>yarn</value>
</property>
<property>
<name>mapreduce.jobhistory.address</name>
<value>wtf-ubuntu:10020</value>
</property>
<property>
<name>mapreduce.jobhistory.webapp.address</name>
<value>wtf-ubuntu:19888</value>
</property>
</configuration>
- 配置 yarn-site.xml
<configuration>
<property>
<name>yarn.nodemanager.aux-services</name>
<value>mapreduce_shuffle</value>
</property>
<property>
<name>yarn.nodemanager.aux-services.mapreduce.shuffle.class</name>
<value>org.apache.hadoop.mapred.ShuffleHandler</value>
</property>
<property>
<name>yarn.resourcemanager.address</name>
<value>wtf-ubuntu:8032</value>
</property>
<property>
<name>yarn.resourcemanager.scheduler.address</name>
<value>wtf-ubuntu:8030</value>
</property>
<property>
<name>yarn.resourcemanager.resource-tracker.address</name>
<value>wtf-ubuntu:8035</value>
</property>
<property>
<name>yarn.resourcemanager.admin.address</name>
<value>wtf-ubuntu:8033</value>
</property>
<property>
<name>yarn.resourcemanager.webapp.address</name>
<value>wtf-ubuntu:8088</value>
</property>
</configuration>
启动
- 格式化NameNode
./bin/hdfs namenode -format
- 操作HDFS
./sbin/start-dfs.sh
./sbin/stop-dfs.sh
./sbin/start-yarn.sh
./sbin/stop-yarn.sh
./bin/hdfs dfsadmin -report
./bin/hadoop fs -mkdir /tmp
./bin/hadoop fs -mkdir /tmp/input
./bin/hadoop fs -ls /tmp/
./bin/hadoop fs -put logs/ /tmp
./bin/hadoop fs -ls /tmp/
./bin/hadoop fs -cat /tmp/logs/
./bin/hadoop fs -cat /tmp/logs/hadoop-hdfs-namenode-wtf-ubuntu.log
web端查看
http://192.168.153.130:50070/