二 hadoop与Spark环境搭建（集群版)

Spark下载地址
JDK: http://www.oracle.com/technetwork/java/javase/downloads/jdk8-downloads-2133151.html
adoop2.6.5：http://archive.apache.org/dist/hadoop/common/hadoop-2.6.5/
Scala2.11.8：https://www.scala-lang.org/download/all.html
Spark2.2.0：http://archive.apache.org/dist/spark/spark-2.2.0/

因为Spark默认存储机制是hdfs






Hadoop集群手动安装部署
所有机器都需要操作

1关闭防火墙
[root@testceph ~]# vi /etc/selinux/config
SELINUX=disabled
2关闭firewalld端口无法访问的事情是经常发生的
systemctl status firewalld
systemctl stop firewalld
systemctl disable firewalld
systemctl status firewalld
firewall-cmd --state

3[root@testceph ~]# cat /etc/hostname
testceph
4[root@testceph ~]# cat /etc/hosts
127.0.0.1   localhost localhost.localdomain localhost4 localhost4.localdomain4
::1         localhost localhost.localdomain localhost6 localhost6.localdomain6
192.168.229.133 testceph
192.168.229.130 redis1
192.168.229.131 redis2
[root@testceph ~]#


5ntp
[root@testceph ~]# yum install ntp -y
[root@testceph ~]# systemctl enable ntpd.service
[root@testceph ~]# systemctl start ntpd.service
[root@testceph ~]# ntpdate 0.centos.pool.ntp.org
6无密钥通信
所有机器之间都要无密钥通信
7用户和root之间不需要密码切换
[root@testceph ~]#echo "hadoop ALL=(ALL) NOPASSWD: ALL" >> /etc/sudoers

二配置JAVA JDK
[root@testceph ~]# tar -zxvf jdk-8u171-linux-x64.tar.gz -C /usr/local/

[root@testceph ~]# vi /etc/profile
export JAVA_HOME=/usr/local/jdk1.8.0_171
export JAVA_HOME=/usr/local/jdk1.8.0_171/jre
export CLASSPATH=.:$JAVA_HOME/lib/dt.jar:$JAVA_HOME/lib/tools.jar:$JRE_HOME/lib
export PATH=$JAVA_HOME/bin:$JRE_HOME/bin:$PATH
[root@testceph ~]# source /etc/profile
[root@testceph ~]# scp -r /usr/local/jdk1.8.0_171 root@redis1:/usr/local/
[root@testceph ~]# scp -r /usr/local/jdk1.8.0_171 root@redis2:/usr/local/
[root@testceph ~]# scp -r /etc/profile root@redis1:/etc/profile
[root@testceph ~]# scp -r /etc/profile root@redis2:/etc/profile
[root@testceph ~]#

三解压修改配置文件
[root@testceph ~]# tar -xvf hadoop-2.6.5.tar -C /usr/local/
[root@testceph ~]# vi /usr/local/hadoop-2.6.5/etc/hadoop/hadoop-env.sh
25  # export JAVA_HOME=${JAVA_HOME}
26  export JAVA_HOME=/usr/local/jdk1.8.0_171/
[root@testceph ~]# vi /usr/local/hadoop-2.6.5/etc/hadoop/core-site.xml
19 <configuration>
20 <property>
21 <name>fs.defaultFS</name>
22 <value>hdfs://testceph:9000</value>
23 </property>
24 </configuration>

[root@testceph ~]# vi /usr/local/hadoop-2.6.5/etc/hadoop/hdfs-site.xml
<configuration>
<property>
        <name>dfs.replication</name>
        <value>1</value>
</property>
</configuration>

[root@testceph ~]# cat /usr/local/hadoop-2.6.5/etc/hadoop/yarn-site.xml
<configuration>

<!-- Site specific YARN configuration properties -->
<property>
    <name>yarn.nodemanager.aux-services</name>
    <value>mapreduce_shuffle</value>
</property>
<property>
    <name>yarn.resourcemanager.address</name>
    <value>testceph:8032</value>
</property>
<property>
    <name>yarn.resourcemanager.scheduler.address</name>
    <value>testceph:8030</value>
</property>
<property>
    <name>yarn.resourcemanager.resource-tracker.address</name>
    <value>testceph:8031</value>
</property>
<property>
    <name>yarn.resourcemanager.admin.address</name>
    <value>testceph:8033</value>
</property>
<property>
    <name>yarn.resourcemanager.webapp.address</name>
    <value>testceph:8088</value>
</property>
<property>
    <name>yarn.nodemanager.pmem-check-enabled</name>
    <value>false</value>
</property>
<property>
    <name>yarn.nodemanager.vmem-check-enabled</name>
    <value>false</value>
</property>
</configuration>
[root@testceph ~]#

配置slaves节点
[root@testceph ~]# cat /usr/local/hadoop-2.6.5/etc/hadoop/slaves
testceph
redis1
redis2
[root@testceph ~]#
四拷贝hadoop软件包到其它节点
[root@testceph ~]# scp -r /usr/local/hadoop-2.6.5 root@redis1:/usr/local/
[root@testceph ~]# scp -r /usr/local/hadoop-2.6.5 root@redis2:/usr/local/

[root@testceph ~]# vi /etc/profile
export HADOOP_HOME=/usr/local/hadoop-2.6.5
export PATH=$PATH:$HADOOP_HOME/bin:$HADOOP_HOME/sbin
[root@testceph ~]# scp /etc/profile root@redis1:/etc/profile
[root@testceph ~]# scp /etc/profile root@redis2:/etc/profile
[root@testceph ~]# source /etc/profile
[root@testceph ~]# which hdfs


五安装
环境变量/etc/profile所有修改配置如下

export JAVA_HOME=/usr/local/jdk1.8.0_171
export JRE_HOME=/usr/local/jdk1.8.0_171/jre
export CLASSPATH=.:$JAVA_HOME/lib/dt.jar:$JAVA_HOME/lib/tools.jar:$JRE_HOME/lib
export PATH=$JAVA_HOME/bin:$JRE_HOME/bin:$PATH

export HADOOP_HOME=/usr/local/hadoop-2.6.5
export PATH=$PATH:$HADOOP_HOME/bin:$HADOOP_HOME/sbin


在主节点testceph上执行以下命令格式化hdfs
[root@testceph ~]# hdfs namenode -format
[root@testceph ~]# echo $?
启动hdfs,在testceph上执行以下操作
[root@testceph ~]# cd /usr/local/hadoop-2.6.5/sbin/
[root@testceph sbin]# ./start-dfs.sh

六验证hdfs是否安装成功
[root@testceph ~]# jps
4801 DataNode
4961 SecondaryNameNode
5450 Jps
4667 NameNode

[root@redis1 ~]# jps
11745 Jps
11498 DataNode

[root@redis2 ~]# jps
12325 DataNode
12613 Jps
[root@redis2 ~]#

是在主节点上使用hdfs dfsadmin -report来观察集群配置情况

网址验证
http://192.168.229.133:50070/dfshealth.html#tab-overview


https://www.linuxidc.com/Linux/2018-06/152795.htm




安装scala
[root@testceph ~]# tar -xvf scala-2.11.8.tar -C /usr/local/
[root@testceph ~]# scp -r /usr/local/scala-2.11.8 root@redis1:/usr/local/
[root@testceph ~]# scp -r /usr/local/scala-2.11.8 root@redis2:/usr/local/
[root@testceph ~]# vi /etc/profile
export SCALA_HOME=/usr/local/scala-2.11.8
export PATH=$PATH:$SCALA_HOME/bin

[root@testceph ~]# source /etc/profile
[root@testceph ~]# which scala
/usr/local/scala-2.11.8/bin/scala
[root@testceph ~]# scala -version
Scala code runner version 2.11.8 -- Copyright 2002-2016, LAMP/EPFL
[root@testceph ~]# scala
Welcome to Scala 2.11.8 (Java HotSpot(TM) 64-Bit Server VM, Java 1.8.0_171).
Type in expressions for evaluation. Or try :help.

scala> println("hello scala!")
hello scala!

scala> :q
[root@testceph ~]#

安装Spark
[root@testceph ~]# tar -xvf spark-2.2.0-bin-hadoop2.6.tgz -C /usr/local/
[root@testceph ~]# cd /usr/local/spark-2.2.0-bin-hadoop2.6/conf/
[root@testceph conf]# cp slaves.template slaves
[root@testceph conf]# vi slaves
# limitations under the License.
#

# A Spark Worker will be started on each of the machines listed below.
testceph
redis1
redis2

[root@testceph conf]# cp spark-env.sh.template spark-env.sh
[root@testceph conf]# tail -f spark-env.sh
export JAVA_HOME=/usr/local/jdk1.8.0_171
export HADOOP_HOME=/usr/local/hadoop-2.6.5
export HADOOP_CONF_DIR=$HADOOP_HOME/etc/hadoop
export SPARK_MASTER_HOST=testceph
export SPARK_MASTER_PORT=7077

[root@testceph ~]# cd
[root@testceph ~]# scp -r /usr/local/spark-2.2.0-bin-hadoop2.6 root@redis1:/usr/local/
[root@testceph ~]# scp -r /usr/local/spark-2.2.0-bin-hadoop2.6 root@redis2:/usr/local/

[root@testceph local]# vi /etc/profile
export SPARK_HOME=/usr/local/spark-2.2.0-bin-hadoop2.6
export PATH=$PATH:${SPARK_HOME}/bin
[root@testceph local]# cd
[root@testceph ~]# scp /etc/profile root@redis1:/etc/profile
profile                                                                                                                                 100% 2452     2.6MB/s   00:00
[root@testceph ~]# scp /etc/profile root@redis2:/etc/profile
profile                                                                                                                                 100% 2452     2.4MB/s   00:00
[root@testceph ~]#

启动Spark集群
[root@testceph ~]# cd /usr/local/spark-2.2.0-bin-hadoop2.6/sbin/
[root@testceph sbin]# ./start-all.sh
starting org.apache.spark.deploy.master.Master, logging to /usr/local/spark-2.2.0-bin-hadoop2.6/logs/spark-root-org.apache.spark.deploy.master.Master-1-testceph.out
redis1: starting org.apache.spark.deploy.worker.Worker, logging to /usr/local/spark-2.2.0-bin-hadoop2.6/logs/spark-root-org.apache.spark.deploy.worker.Worker-1-redis1.out
testceph: starting org.apache.spark.deploy.worker.Worker, logging to /usr/local/spark-2.2.0-bin-hadoop2.6/logs/spark-root-org.apache.spark.deploy.worker.Worker-1-testceph.out
redis2: starting org.apache.spark.deploy.worker.Worker, logging to /usr/local/spark-2.2.0-bin-hadoop2.6/logs/spark-root-org.apache.spark.deploy.worker.Worker-1-redis2.out
[root@testceph sbin]#

验证Spark集群
http://192.168.229.133:8080