Hadoop2.0 YARN cloudra4.4.0安装配置

最新推荐文章于 2024-07-26 11:52:59 发布

hjseo_seg

最新推荐文章于 2024-07-26 11:52:59 发布

阅读量510

点赞数

分类专栏： java 文章标签： hadoop hdfs 大数据搜索引擎运维

本文链接：https://blog.csdn.net/hjseo_seg/article/details/126366523

版权

java 专栏收录该内容

286 篇文章 14 订阅

订阅专栏

1
2
3

hadoop@hadoop-virtual-machine:~$ cat /etc/hostname
yard02
hadoop@hadoop-virtual-machine:~$

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15

hadoop@hadoop-virtual-machine:~$ cat /etc/hosts
127.0.0.1 localhost
127.0.1.1 hadoop-virtual-machine

The following lines are desirable for IPv6 capable hosts

::1 ip6-localhost ip6-loopback
fe00::0 ip6-localnet
ff00::0 ip6-mcastprefix
ff02::1 ip6-allnodes
ff02::2 ip6-allrouters
192.168.137.2 yard02
192.168.137.3 yard03
192.168.137.4 yard04
192.168.137.5 yard05
hadoop@hadoop-virtual-machine:~$

core-site.xml —-conf/hadoop

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18

<property>
<name>fs.defaultFS</name>
<value>hdfs://yard02</value>
</property>
<property>
<name>fs.trash.interval</name>
<value>10080</value>
</property>
<property>
<name>fs.trash.checkpoint.interval</name>
<value>10080</value>
</property>
</configuration>

hdfs-site.xml

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22

<configuration>
<property>
<name>dfs.replication</name>
<value>3</value>
</property>
<property>
<name>hadoop.tmp.dir</name>
<value>/home/hadoop/bigdata/hadoop-2.0.0-cdh4.4.0/data/hadoop-${user.name}</value>
</property>
<property>
<name>dfs.namenode.http-address</name>
<value>yard02:50070</value>
</property>
<property>
<name>dfs.namenode.secondary.http-address</name>
<value>yard03:50090</value>
</property>
<property>
<name>dfs.webhdfs.enabled</name>
<value>true</value>
</property>
</configuration>

yarn-site.xml

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57

<property>
<name>yarn.resourcemanager.resource-tracker.address</name>
<value>yard02:8031</value>
</property>
<property>
<name>yarn.resourcemanager.address</name>
<value>yard02:8032</value>
</property>
<property>
<name>yarn.resourcemanager.scheduler.address</name>
<value>yard02:8030</value>
</property>
<property>
<name>yarn.resourcemanager.admin.address</name>
<value>yard02:8033</value>
</property>
<property>
<name>yarn.resourcemanager.webapp.address</name>
<value>yard02:8088</value>
</property>
<property>
<description>Classpath for typical applications.</description>
<name>yarn.application.classpath</name>
<value> $HADOOP\_CONF\_DIR,$ HADOOP_COMMON_HOME/share/hadoop/common/*,
$HADOOP_COMMON_HOME/share/hadoop/common/lib/*,
$KaTeX parse error: Undefined control sequence: \* at position 38: …re/hadoop/hdfs/\̲*̲,$ HADOOP_HDFS_HOME/share/hadoop/hdfs/lib/*,
$KaTeX parse error: Undefined control sequence: \* at position 30: …re/hadoop/yarn/\̲*̲,$ YARN_HOME/share/hadoop/yarn/lib/*,
$KaTeX parse error: Undefined control sequence: \* at position 35: …doop/mapreduce/\̲*̲,$ YARN_HOME/share/hadoop/mapreduce/lib/*</value>
</property>
<property>
<name>yarn.nodemanager.aux-services</name>
<value>mapreduce.shuffle</value>
</property>
<property>
<name>yarn.nodemanager.aux-services.mapreduce.shuffle.class</name>
<value>org.apache.hadoop.mapred.ShuffleHandler</value>
</property>
<property>
<name>yarn.nodemanager.local-dirs</name>
<value>/home/hadoop/bigdata/hadoop-2.0.0-cdh4.4.0/data/yarn/local</value>
</property>
<property>
<name>yarn.nodemanager.log-dirs</name>
<value>/home/hadoop/bigdata/hadoop-2.0.0-cdh4.4.0/data/yarn/logs</value>
</property>
<property>
<description>Where to aggregate logs</description>
<name>yarn.nodemanager.remote-app-log-dir</name>
<value>/home/hadoop/bigdata/hadoop-2.0.0-cdh4.4.0/data/yarn/logs</value>
</property>
<property>
<name>yarn.app.mapreduce.am.staging-dir</name>
<value>/home/hadoop/bigdata/hadoop-2.0.0-cdh4.4.0</value>
</property>
</configuration>

mapred-site.xml

1
2
3
4
5
6
7
8
9
10
11
12
13
14

<configuration>
<property>
<name>mapreduce.framework.name</name>
<value>yarn</value>
</property>
<property>
<name>mapreduce.jobhistory.address</name>
<value>yard2:10020</value>
</property>
<property>
<name>mapreduce.jobhistory.webapp.address</name>
<value>yard02:19888</value>
</property>
</configuration>

masters

1
2

yard02
yard03

slaves

1
2
3
4

yard02
yard03
yard04
yard05

.bashrc

1
2
3
4
5
6
7
8
9
10

export HADOOP_HOME=/home/hadoop/bigdata/hadoop-2.0.0-cdh4.4.0
export HADOOP_MAPRED_HOME= ${HADOOP\_HOME} export HADOOP\_COMMON\_HOME=$ {HADOOP_HOME}
export HADOOP_HDFS_HOME= ${HADOOP\_HOME} export YARN\_HOME=$ {HADOOP_HOME}
export HADOOP_YARN_HOME= ${HADOOP\_HOME} export HADOOP\_CONF\_DIR=$ {HADOOP_HOME}/etc/hadoop
export HDFS_CONF_DIR= ${HADOOP\_HOME}/etc/hadoop export YARN\_CONF\_DIR=$ {HADOOP_HOME}/etc/hadoop
export PATH= $P A T H :$ HOME/bin: $JAVA\_HOME/bin:$ HADOOP_HOME/sbin

同步个配置文件

1
2
3
4
5
6
7

scp -r /home/hadoop/bigdata/hadoop-2.0.0-cdh4.4.0/* hadoop@yard03:/home/hadoop/bigdata/hadoop-2.0.0-cdh4.4.0/
scp -r /home/hadoop/bigdata/hadoop-2.0.0-cdh4.4.0/* hadoop@yard04:/home/hadoop/bigdata/hadoop-2.0.0-cdh4.4.0/
scp -r /home/hadoop/bigdata/hadoop-2.0.0-cdh4.4.0/* hadoop@yard05:/home/hadoop/bigdata/hadoop-2.0.0-cdh4.4.0/

scp /home/hadoop/.bashrc hadoop@yard03:/home/hadoop
scp /home/hadoop/.bashrc hadoop@yard04:/home/hadoop
scp /home/hadoop/.bashrc hadoop@yard05:/home/hadoop

关于ssh无密码还是这个屡试不爽

1
2
3
4
5
6
7

ssh-keygen -t rsa
cp id_rsa.pub authorized_keys //本地也要哦
ssh localhost //确保本地可以无密码登陆

scp authorized_keys hadoop@yard03:/home/hadoop/.ssh
scp authorized_keys hadoop@yard04:/home/hadoop/.ssh
scp authorized_keys hadoop@yard05:/home/hadoop/.ssh

这个启动基本在sbin目录了，
格式化：

hadoop namenode -format

启动

start-all.sh

启动hdfs：

start-dfs.sh

启动mapreduce

start-yarn.sh

启动historyserver

mr-jobhistory-daemon.sh start historyserver

WordCount 测试
./hadoop fs -put /etc/profile /user/hadoop/input
./hadoop jar …/share/hadoop/mapreduce/hadoop-mapreduce-examples-2.0.0-cdh4.4.0.jar wordcount input output
输出结果
访问：
http://yard02:50070/
http://yard03:50090
http://yard02:8088/cluster
http://yard02:8088/cluster/apps
http://yard02:8088/cluster/nodes
http://yard02:8088/cluster/scheduler