HDFS 单机搭建

HDFS 单机搭建

标签 ： Java hadoop hdfs

---

安装

• 请点击下载hadoop-2.6.4

• 解压到 /home/hdfs/hadoop

useradd hdfs // 创建hdfs用户
passwd hdfs // 设置hdfs用户的登陆密码
su hdfs // 切换到hdfs用户登陆
cd ~ // 进入hdfs的家目录
rz // 上传windows本地的文件到此家目录
tar -zxvf hadoop-2.6.4.tar.gz // 解压安装包
mv hadoop-2.6.4 hadoop // 重命名解压后的文件名

• hdfs机器免密验证设置

cd ~/.ssh
ssh-keygen -t rsa // 一路摁回车
id_rsa // 私钥
id_rsa.pub // 公钥
cat id_rsa.pub >> authorized_keys // 在master上将公钥放到authorized_keys(已认证的keys)里
sudo scp authorized_keys hdfs@192.168.153.129:~/.ssh 
// 将master上的authorized_keys放到其他(需要免密的节点)linux的~/.ssh目录下
sudo scp authorized_keys hdfs@192.168.153.128:~/.ssh
// 将master上的authorized_keys放到其他(需要免密的节点)linux的~/.ssh目录下
chmod 644 authorized_keys // 修改钥的权限

• 设置Java环境变量

vi ~/bashrc
export JAVA_HOME=/java_install_dir/jdk1.8.0_91
export JRE_HOME=/java_install_dir/jdk1.8.0_91/jre
export CLASSPATH=.:$CLASSPATH:$JAVA_HOME/lib:$JRE_HOME/lib
export PATH=$PATH:$JAVA_HOME/bin:$JRE_HOME/bin
source ~/.bashrc // 让配置生效
echo $JAVA_HOME
/java_install_dir/jdk1.8.0_91

配置 *.xml

• 配置hadoop-env.sh

export JAVA_HOME=/java_install_dir/jdk1.8.0_91

• 配置 yarn-env.sh

export JAVA_HOME=/java_install_dir/jdk1.8.0_91

• 配置 core-site.xml

<configuration>
    <property>
        <name>fs.defaultFS</name>
        <value>hdfs://wtf-ubuntu:9000</value>
    </property>
    <property>
        <name>io.file.buffer.size</name>
        <value>131072</value>
    </property>
    <property>
        <name>hadoop.tmp.dir</name>
        <value>file:/home/hdfs/hadoop/tmp</value>
        <description>Abasefor other temporary directories.</description>
    </property>
    <property>
        <name>hadoop.proxyuser.spark.hosts</name>
        <value>*</value>
    </property>
    <property>
        <name>hadoop.proxyuser.spark.groups</name>
        <value>*</value>
    </property>
</configuration>

• 配置hdfs-site.xml

<configuration>
    <property>
        <name>dfs.namenode.secondary.http-address</name>
        <value>wtf-ubuntu:9001</value>
    </property>

    <property>
        <name>dfs.namenode.name.dir</name>
        <value>file:/home/hdfs/hadoop/name</value>
    </property>

    <property>
        <name>dfs.datanode.data.dir</name>
        <value>file:/home/hdfs/hadoop/data</value>
    </property>

    <property>
        <name>dfs.replication</name>
        <value>3</value>
    </property>

    <property>
        <name>dfs.webhdfs.enabled</name>
        <value>true</value>
    </property>

</configuration>

• 配置 mapred-site.xml 文件

cp mapred-site.xml.template mapred-site.xml // 复制一份模板重命名为 mapred-site.xml
<configuration>
    <property>
        <name>mapreduce.framework.name</name>
        <value>yarn</value>
    </property>
    <property>
        <name>mapreduce.jobhistory.address</name>
        <value>wtf-ubuntu:10020</value>
    </property>
    <property>
        <name>mapreduce.jobhistory.webapp.address</name>
        <value>wtf-ubuntu:19888</value>
    </property>
</configuration>

• 配置 yarn-site.xml

<configuration>
    <property>
        <name>yarn.nodemanager.aux-services</name>
        <value>mapreduce_shuffle</value>
    </property>
    <property>
        <name>yarn.nodemanager.aux-services.mapreduce.shuffle.class</name>
        <value>org.apache.hadoop.mapred.ShuffleHandler</value>
    </property>
    <property>
        <name>yarn.resourcemanager.address</name>
        <value>wtf-ubuntu:8032</value>
    </property>
    <property>
        <name>yarn.resourcemanager.scheduler.address</name>
        <value>wtf-ubuntu:8030</value>
    </property>
    <property>
        <name>yarn.resourcemanager.resource-tracker.address</name>
        <value>wtf-ubuntu:8035</value>
    </property>
    <property>
        <name>yarn.resourcemanager.admin.address</name>
        <value>wtf-ubuntu:8033</value>
    </property>
    <property>
        <name>yarn.resourcemanager.webapp.address</name>
        <value>wtf-ubuntu:8088</value>
    </property>

</configuration>

启动

• 格式化NameNode

./bin/hdfs namenode -format

• 操作HDFS

./sbin/start-dfs.sh
./sbin/stop-dfs.sh 
./sbin/start-yarn.sh 
./sbin/stop-yarn.sh
./bin/hdfs dfsadmin -report

./bin/hadoop fs  -mkdir /tmp
./bin/hadoop fs  -mkdir /tmp/input
./bin/hadoop fs  -ls /tmp/
./bin/hadoop fs  -put logs/ /tmp
./bin/hadoop fs  -ls /tmp/
./bin/hadoop fs  -cat /tmp/logs/
./bin/hadoop fs  -cat /tmp/logs/hadoop-hdfs-namenode-wtf-ubuntu.log

web端查看

http://192.168.153.130:50070/