实验环境
Hadoop(3.2.0) 伪分布式wordcount
hdfs dfs -mkdir -p /user/yourUserName
hdfs dfs -mkdir input1
hdfs dfs -copyFromLocal ./input/* input1
hdfs dfs -ls input1
hdfs dfs -rmr output
bin/hadoop jar share/hadoop/mapreduce/hadoop-mapreduce-examples-3.2.0.jar grep input1 output 'dfs[a-z.]+'
hdfs dfs -cat output/part*
免密登陆
ssh-keygen -t rsa -P '' -f ~/.ssh/id_rsa
cat ~/.ssh/id_rsa.pub >> ~/.ssh/authorized_keys
chmod 0600 ~/.ssh/authorized_keys
ssh权限问题
[admin@iz2ze22ghnnsejh781lqrqz ~]$ sudo chmod 600 ~/.ssh/id_rsa
[admin@iz2ze22ghnnsejh781lqrqz ~]$ sudo chmod 600 ~/.ssh/id_rsa.pub
[admin@iz2ze22ghnnsejh781lqrqz ~]$ sudo chmod 755 ~/.ssh
[admin@iz2ze22ghnnsejh781lqrqz ~]$
[admin@iz2ze22ghnnsejh781lqrqz ~]$ start-dfs.sh
Starting namenodes on [localhost]
Starting datanodes
Starting secondary namenodes [iz2ze22ghnnsejh781lqrqz]
[admin@iz2ze22ghnnsejh781lqrqz ~]$ jps
11649 SecondaryNameNode
11764 Jps
11478 DataNode
[admin@iz2ze22ghnnsejh781lqrqz ~]$
input1文件夹
Found 9 items
-rw-r--r-- 1 hadoop supergroup 8260 2019-09-16 15:53 input1/capacity-scheduler.xml
-rw-r--r-- 1 hadoop supergroup 1110 2019-09-16 15:53 input1/core-site.xml
-rw-r--r-- 1 hadoop supergroup 11392 2019-09-16 15:53 input1/hadoop-policy.xml
-rw-r--r-- 1 hadoop supergroup 1293 2019-09-16 15:53 input1/hdfs-site.xml
-rw-r--r-- 1 hadoop supergroup 620 2019-09-16 15:53 input1/httpfs-site.xml
-rw-r--r-- 1 hadoop supergroup 3518 2019-09-16 15:53 input1/kms-acls.xml
-rw-r--r-- 1 hadoop supergroup 682 2019-09-16 15:53 input1/kms-site.xml
-rw-r--r-- 1 hadoop supergroup 758 2019-09-16 15:53 input1/mapred-site.xml
-rw-r--r-- 1 hadoop supergroup 690 2019-09-16 15:53 input1/yarn-site.xml
词频统计结果
1 dfsadmin
1 dfs.replication
1 dfs.name.dir
1 dfs.data.dir
hdfs-site.xml
<configuration>
<property>
<name>dfs.name.dir</name>
<value>/usr/hadoop/hdfs/name</value>
<description>namenode上存储hdfs名字空间元数据 </description>
</property>
<property>
<name>dfs.data.dir</name>
<value>/usr/hadoop/hdfs/data</value>
<description>datanode上数据块的物理存储位置</description>
</property>
<!-- 设置hdfs副本数量 -->
<property>
<name>dfs.replication</name>
<value>1</value>
</property>
</configuration>
core-site.xml
<configuration>
<!-- 指定HDFS namenode 的通信地址 -->
<property>
<name>fs.defaultFS</name>
<value>hdfs://localhost:9000</value>
</property>
<!-- 指定hadoop运行时产生文件的存储路径 -->
<property>
<name>hadoop.tmp.dir</name>
<value>/usr/local/hadoop/hadoop-3.2.0/tmp</value>
</property>
</configuration>