安装版本
04 August, 2017: Release 2.7.4
注意事项
- hostname 不要加特殊字符串,只用字母和数字(在停止的时候一直读取到的用户名是小写的,具体原因不清楚)
- 本次安装中 master 是主机 11.11.11.4,slaves 是主机 11.11.11.3,11.11.11.9
启动之后进程
slaves
[root@host3 hadoop-2.7.4]# jps -l 1225 org.apache.hadoop.hdfs.server.datanode.DataNode 1343 org.apache.hadoop.yarn.server.nodemanager.NodeManager [root@host9 hadoop-2.7.4]# jps -l 1237 org.apache.hadoop.hdfs.server.datanode.DataNode 1355 org.apache.hadoop.yarn.server.nodemanager.NodeManager
master
[root@host4 hadoop-2.7.4]# jps -l 1410 org.apache.hadoop.yarn.server.resourcemanager.ResourceManager 1256 org.apache.hadoop.hdfs.server.namenode.NameNode
- web管理URL http://11.11.11.4:50070
- 启动和停止脚本的运行都是在master主机上运行,不在slaves所在主机做任何命令
- stop-all.sh
[root@host4 hadoop-2.7.4]# ./sbin/stop-all.sh
This script is Deprecated. Instead use stop-dfs.sh and stop-yarn.sh
Stopping namenodes on [host4]
host4: stopping namenode
11.11.11.3: stopping datanode
11.11.11.9: stopping datanode
Stopping secondary namenodes [host4]
host4: stopping secondarynamenode
stopping yarn daemons
stopping resourcemanager
11.11.11.9: stopping nodemanager
11.11.11.3: stopping nodemanager
no proxyserver to stop
- start-all.sh
[root@host4 hadoop-2.7.4]# ./sbin/start-all.sh
This script is Deprecated. Instead use start-dfs.sh and start-yarn.sh
Starting namenodes on [host4]
host4: starting namenode, logging to /usr/local/hadoop-2.7.4/logs/hadoop-root-namenode-host4.out
11.11.11.9: starting datanode, logging to /usr/local/hadoop-2.7.4/logs/hadoop-root-datanode-host9.out
11.11.11.3: starting datanode, logging to /usr/local/hadoop-2.7.4/logs/hadoop-root-datanode-host3.out
Starting secondary namenodes [host4]
host4: starting secondarynamenode, logging to /usr/local/hadoop-2.7.4/logs/hadoop-root-secondarynamenode-host4.out
starting yarn daemons
starting resourcemanager, logging to /usr/local/hadoop-2.7.4/logs/yarn-root-resourcemanager-host4.out
11.11.11.3: starting nodemanager, logging to /usr/local/hadoop-2.7.4/logs/yarn-root-nodemanager-host3.out
11.11.11.9: starting nodemanager, logging to /usr/local/hadoop-2.7.4/logs/yarn-root-nodemanager-host9.out
- jps
#master
[root@host4 hadoop-2.7.4]# jps -l
4706 sun.tools.jps.Jps
4083 org.apache.hadoop.hdfs.server.namenode.NameNode
4275 org.apache.hadoop.hdfs.server.namenode.SecondaryNameNode
4440 org.apache.hadoop.yarn.server.resourcemanager.ResourceManager
1710 org.apache.zookeeper.server.quorum.QuorumPeerMain
#slave
[root@host9 hadoop-2.7.4]# jps -l
2291 org.apache.hadoop.yarn.server.nodemanager.NodeManager
2438 sun.tools.jps.Jps
2185 org.apache.hadoop.hdfs.server.datanode.DataNode
1518 org.apache.zookeeper.server.quorum.QuorumPeerMain
#slave
[root@host3 hadoop-2.7.4]# jps -l
2161 org.apache.hadoop.hdfs.server.datanode.DataNode
1493 org.apache.zookeeper.server.quorum.QuorumPeerMain
2267 org.apache.hadoop.yarn.server.nodemanager.NodeManager
2399 sun.tools.jps.Jps
- netstat
#slave
[root@host3 hadoop-2.7.4]# netstat -anto
Active Internet connections (servers and established)
Proto Recv-Q Send-Q Local Address Foreign Address State Timer
tcp 0 0 0.0.0.0:22 0.0.0.0:* LISTEN off (0.00/0/0)
tcp 0 0 127.0.0.1:25 0.0.0.0:* LISTEN off (0.00/0/0)
tcp 0 0 0.0.0.0:50010 0.0.0.0:* LISTEN off (0.00/0/0)
tcp 0 0 0.0.0.0:50075 0.0.0.0:* LISTEN off (0.00/0/0)
tcp 0 0 127.0.0.1:37985 0.0.0.0:* LISTEN off (0.00/0/0)
tcp 0 0 0.0.0.0:50020 0.0.0.0:* LISTEN off (0.00/0/0)
tcp 0 0 11.11.11.3:22 11.11.11.4:43435 ESTABLISHED keepalive (5907.23/0/0)
tcp 0 0 11.11.11.3:59127 11.11.11.4:9000 ESTABLISHED keepalive (6994.07/0/0)
tcp 0 0 :::36914 :::* LISTEN off (0.00/0/0)
tcp 0 0 :::22 :::* LISTEN off (0.00/0/0)
tcp 0 0 ::1:25 :::* LISTEN off (0.00/0/0)
tcp 0 0 :::13562 :::* LISTEN off (0.00/0/0)
tcp 0 0 :::60514 :::* LISTEN off (0.00/0/0)
tcp 0 0 ::ffff:11.11.11.3:2181 :::* LISTEN off (0.00/0/0)
tcp 0 0 :::8040 :::* LISTEN off (0.00/0/0)
tcp 0 0 :::8042 :::* LISTEN off (0.00/0/0)
tcp 0 0 ::ffff:11.11.11.3:3888 :::* LISTEN off (0.00/0/0)
tcp 0 0 ::ffff:11.11.11.3:3888 ::ffff:11.11.11.9:50710 ESTABLISHED off (0.00/0/0)
tcp 0 0 ::ffff:11.11.11.3:3888 ::ffff:11.11.11.4:41540 ESTABLISHED off (0.00/0/0)
tcp 0 0 ::ffff:11.11.11.3:51059 ::ffff:11.11.11.4:2888 ESTABLISHED off (0.00/0/0)
tcp 0 0 ::ffff:11.11.11.3:50958 ::ffff:11.11.11.4:18025 ESTABLISHED keepalive (7007.43/0/0)
#slave
[root@host9 hadoop-2.7.4]# netstat -anto
Active Internet connections (servers and established)
Proto Recv-Q Send-Q Local Address Foreign Address State Timer
tcp 0 0 127.0.0.1:53489 0.0.0.0:* LISTEN off (0.00/0/0)
tcp 0 0 0.0.0.0:22 0.0.0.0:* LISTEN off (0.00/0/0)
tcp 0 0 127.0.0.1:25 0.0.0.0:* LISTEN off (0.00/0/0)
tcp 0 0 0.0.0.0:50010 0.0.0.0:* LISTEN off (0.00/0/0)
tcp 0 0 0.0.0.0:50075 0.0.0.0:* LISTEN off (0.00/0/0)
tcp 0 0 0.0.0.0:50020 0.0.0.0:* LISTEN off (0.00/0/0)
tcp 0 0 11.11.11.9:22 11.11.11.4:33271 ESTABLISHED keepalive (5833.68/0/0)
tcp 0 0 11.11.11.9:45797 11.11.11.4:9000 ESTABLISHED keepalive (6929.57/0/0)
tcp 0 0 :::22 :::* LISTEN off (0.00/0/0)
tcp 0 0 :::36729 :::* LISTEN off (0.00/0/0)
tcp 0 0 ::1:25 :::* LISTEN off (0.00/0/0)
tcp 0 0 :::13562 :::* LISTEN off (0.00/0/0)
tcp 0 0 ::ffff:11.11.11.9:2181 :::* LISTEN off (0.00/0/0)
tcp 0 0 :::8040 :::* LISTEN off (0.00/0/0)
tcp 0 0 :::8042 :::* LISTEN off (0.00/0/0)
tcp 0 0 :::42572 :::* LISTEN off (0.00/0/0)
tcp 0 0 ::ffff:11.11.11.9:3888 :::* LISTEN off (0.00/0/0)
tcp 0 0 ::ffff:11.11.11.9:35379 ::ffff:11.11.11.4:2888 ESTABLISHED off (0.00/0/0)
tcp 0 0 ::ffff:11.11.11.9:57459 ::ffff:11.11.11.4:18025 ESTABLISHED keepalive (6942.69/0/0)
tcp 0 0 ::ffff:11.11.11.9:52517 ::ffff:11.11.11.4:3888 ESTABLISHED off (0.00/0/0)
tcp 0 0 ::ffff:11.11.11.9:50710 ::ffff:11.11.11.3:3888 ESTABLISHED off (0.00/0/0)
#master
[root@host4 hadoop-2.7.4]# netstat -anto
Active Internet connections (servers and established)
Proto Recv-Q Send-Q Local Address Foreign Address State Timer
tcp 0 0 0.0.0.0:50070 0.0.0.0:* LISTEN off (0.00/0/0)
tcp 0 0 0.0.0.0:22 0.0.0.0:* LISTEN off (0.00/0/0)
tcp 0 0 127.0.0.1:25 0.0.0.0:* LISTEN off (0.00/0/0)
tcp 0 0 11.11.11.4:9000 0.0.0.0:* LISTEN off (0.00/0/0)
tcp 0 0 11.11.11.4:50090 0.0.0.0:* LISTEN off (0.00/0/0)
tcp 0 0 11.11.11.4:22 192.168.0.99:54470 ESTABLISHED keepalive (5769.92/0/0)
tcp 0 0 11.11.11.4:9000 11.11.11.9:45797 ESTABLISHED keepalive (6909.35/0/0)
tcp 0 0 11.11.11.4:22 192.168.0.99:54809 ESTABLISHED keepalive (6717.90/0/0)
tcp 0 0 11.11.11.4:9000 11.11.11.3:59127 ESTABLISHED keepalive (6909.26/0/0)
tcp 0 0 11.11.11.4:22 192.168.0.99:54472 ESTABLISHED keepalive (5772.03/0/0)
tcp 160 0 11.11.11.4:22 192.168.0.99:54469 ESTABLISHED keepalive (5767.57/0/0)
tcp 0 0 11.11.11.4:33271 11.11.11.9:22 ESTABLISHED keepalive (5813.36/0/0)
tcp 0 0 11.11.11.4:33944 11.11.11.4:9000 TIME_WAIT timewait (25.67/0/0)
tcp 0 0 11.11.11.4:43435 11.11.11.3:22 ESTABLISHED keepalive (5822.36/0/0)
tcp 0 0 :::22 :::* LISTEN off (0.00/0/0)
tcp 0 0 ::ffff:11.11.11.4:18040 :::* LISTEN off (0.00/0/0)
tcp 0 0 ::1:25 :::* LISTEN off (0.00/0/0)
tcp 0 0 :::47674 :::* LISTEN off (0.00/0/0)
tcp 0 0 ::ffff:11.11.11.4:18141 :::* LISTEN off (0.00/0/0)
tcp 0 0 ::ffff:11.11.11.4:2181 :::* LISTEN off (0.00/0/0)
tcp 0 0 ::ffff:11.11.11.4:18088 :::* LISTEN off (0.00/0/0)
tcp 0 0 ::ffff:11.11.11.4:2888 :::* LISTEN off (0.00/0/0)
tcp 0 0 ::ffff:11.11.11.4:18025 :::* LISTEN off (0.00/0/0)
tcp 0 0 ::ffff:11.11.11.4:18030 :::* LISTEN off (0.00/0/0)
tcp 0 0 ::ffff:11.11.11.4:3888 :::* LISTEN off (0.00/0/0)
tcp 0 0 ::ffff:11.11.11.4:3888 ::ffff:11.11.11.9:52517 ESTABLISHED off (0.00/0/0)
tcp 0 0 ::ffff:11.11.11.4:2888 ::ffff:11.11.11.3:51059 ESTABLISHED off (0.00/0/0)
tcp 0 0 ::ffff:11.11.11.4:18025 ::ffff:11.11.11.9:57459 ESTABLISHED keepalive (6922.48/0/0)
tcp 0 0 ::ffff:11.11.11.4:2888 ::ffff:11.11.11.9:35379 ESTABLISHED off (0.00/0/0)
tcp 0 0 ::ffff:11.11.11.4:18025 ::ffff:11.11.11.3:50958 ESTABLISHED keepalive (6922.61/0/0)
tcp 0 0 ::ffff:11.11.11.4:41540 ::ffff:11.11.11.3:3888 ESTABLISHED off (0.00/0/0)
安装步骤
- 安装JDK
- 配置免密登录
- 免密登录即将本机通过ssh-keygen -t rsa 生成的 ~/.ssh/id_rsa.pub 文件追加到远程要登录的 ~/.ssh/authorized_keys文件尾部即可免密登录
- 安装hadoop
- 下载解压
- 配置HADOOP_HOME(/etc/profile,完成之后用source /etc/profile使其生效)
- 配置相关信息
- 启动服务
启动方式
- 格式化namenode(仅在master所在机子)
cd $HADOOP_HOME
./bin/hdfs namenode -format <cluster_name>
- 启动hdfs
cd $HADOOP_HOME
./sbin/hadoop-daemon.sh start namenode
./sbin/hadoop-daemons.sh start datanode
上面两条命令可用用下面一条代替
cd $HADOOP_HOME
./sbin/start-dfs.sh
- 启动yarn
cd $HADOOP_HOME
./sbin/yarn-daemon.sh start resourcemanager
./sbin/yarn-daemons.sh start nodemanager
上面两条命令可用用下面一条代替
cd $HADOOP_HOME
./sbin/start-yarn.sh
停止服务
将启动中的start改成stop即可
配置文件
所有配置文件所在目录 $HADOOP_HOME/etc/hadoop
指定JDK的位置
修改hadoop-env.sh和yarn-env.sh文件中的JAVA_HOME的值为JDK安装目录
slaves
11.11.11.3
11.11.11.4
core-site.xml
<?xml version="1.0" encoding="UTF-8"?>
<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
<!--
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License. See accompanying LICENSE file.
-->
<!-- Put site-specific property overrides in this file. -->
<configuration>
<property>
<name>fs.defaultFS</name>
<value>hdfs://11.11.11.4:9000</value>
</property>
<property>
<name>hadoop.tmp.dir</name>
<value>file:///data/hadoop/tmp</value>
<description>A base for other temporary directories.</description>
</property>
</configuration>
hdfs-site.xml
<?xml version="1.0" encoding="UTF-8"?>
<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
<!--
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License. See accompanying LICENSE file.
-->
<!-- Put site-specific property overrides in this file. -->
<configuration>
<property>
<name>dfs.replication</name>
<value>3</value>
</property>
<property>
<name>dfs.namenode.name.dir</name>
<value>file:///data/hadoop/hdfs/name</value>
<final>true</final>
</property>
<property>
<name>dfs.datanode.data.dir</name>
<value>file:///data/hadoop/hdfs/data</value>
<final>true</final>
</property>
<property>
<name>dfs.namenode.secondary.http-address</name>
<value>11.11.11.4:50090</value>
</property>
<property>
<name>dfs.webhdfs.enabled</name>
<value>true</value>
</property>
</configuration>
mapred-site.xml
<?xml version="1.0"?>
<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
<!--
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License. See accompanying LICENSE file.
-->
<!-- Put site-specific property overrides in this file. -->
<configuration>
<property>
<name>mapreduce.framework.name</name>
<value>yarn</value>
</property>
</configuration>
yarn-site.xml
<?xml version="1.0"?>
<!--
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License. See accompanying LICENSE file.
-->
<configuration>
<!-- Site specific YARN configuration properties -->
<property>
<name>yarn.resourcemanager.address</name>
<value>11.11.11.4:18040</value>
</property>
<property>
<name>yarn.resourcemanager.scheduler.address</name>
<value>11.11.11.4:18030</value>
</property>
<property>
<name>yarn.resourcemanager.webapp.address</name>
<value>11.11.11.4:18088</value>
</property>
<property>
<name>yarn.resourcemanager.resource-tracker.address</name>
<value>11.11.11.4:18025</value>
</property>
<property>
<name>yarn.resourcemanager.admin.address</name>
<value>11.11.11.4:18141</value>
</property>
<property>
<name>yarn.nodemanager.aux-services</name>
<value>mapreduce_shuffle</value>
</property>
<property>
<name>yarn.resourcemanager.hostname</name>
<value>11.11.11.4</value>
</property>
</configuration>