Linux搭建Hadoop jdk Hive Zookeeper HBase Spark

vmware workstation安装

下载地址：https://download3.vmware.com/software/wkst/file/VMware-workstation-full-16.2.2-19200509.exe

VM 网络配置

CentOS网路配置

开机自启网络

# 修改配置文件
vim /etc/sysconfig/network-scripts/ifcfg-ens32

onboot=yes

配置固定IP

配置虚拟网络

# 修改配置文件
vim /etc/sysconfig/network-scripts/ifcfg-ens32

# IP地址大家配一个自己想要配的地址，但是必须是同VMnet8的子网IP在同一网段
IPADDR=192.168.22.131
# 网关，这个值与我们在第四步“NAT（设置S）...”设置的网关一样
GATEWAY=192.168.22.2
# 子网掩码
NETMASK=255.255.255.0
# DNS的值也跟我们第四步“NAT（设置）...”设置的的网关一样
DNS1=192.168.22.2

修改之后的配置文件

重启网络生效

sudo systemctl restart network

# 关闭防火墙
sudo systemctl stop firewalld.service

# 禁止开机启动
sudo systemctl disable firewalld.service

# 开启 sshd 服务
systemctl restart sshd

# 查看版本
cat /etc/redhat-release

# 配置环境变量后，立刻生效
source /etc/profile

主机不能ping通虚拟机

SELinux

状态查看

修改状态

vim /etc/selinux/config

# SELINUX=enforcing
SELINUX=disabled

# 重启机器
reboot

配置ssh

ssh-keygen -t rsa

# 进入目录
cd /root/.ssh

# 把 id_rsa.pub 中的内容写入到 authorized_keys
cat id_rsa.pub > authorized_keys

jdk1.8安装配置

jdk准备

# 进入 /usr/local 目录
cd /usr/local
mkdir java

# 解压 jdk 文件到 /usr/local/java 下
tar -xvf jdk-8u221-linux-x64.tar.gz -C /usr/local/java

jdk环境配置

编辑 vim /etc/profile 文件

export JAVA_HOME=/usr/local/java/jdk1.8.0_221
export CLASSPATH=.:$JAVA_HOME/lib/dt.jar:$JAVA_HOME/lib/tools.jar 
export PATH=$JAVA_HOME/bin:$PATH

使其生效以及查看

# 生效
source /etc/profile

# 查看 
java -version

Hadoop 安装

• master [ 192.168.22.131 ]
• slave1 [ 192.168.22.132 ]
• slave2 [ 192.168.22.133 ]

配置 master

修改主机名

hostnamectl set-hostname master

# 修改 hosts
vim /etc/hosts

192.168.22.131 master
192.168.22.132 slave1
192.168.22.133 slave2

配置 Hadoop

# 解压文件
tar -xvf hadoop-3.2.2.tar.gz -C /opt/

配置环境变量

vim /etc/profile

export HADOOP_HOME=/opt/hadoop-3.2.2

export PATH=$JAVA_HOME/bin:$HADOOP_HOME/bin:$HADOOP_HOME/sbin:$PATH

# 生效
source /etc/profile

创建 hadoop 文件夹

mkdir -p /root/hadoop
mkdir -p /root/hadoop/tmp
mkdir -p /root/hadoop/var
mkdir -p /root/hadoop/dfs
mkdir -p /root/hadoop/dfs/name
mkdir -p /root/hadoop/dfs/data

修改core-site.xml

cd /opt/hadoop-3.2.2/etc/hadoop
vim core-site.xml

<configuration>
        <property>
                <name>hadoop.tmp.dir</name>
                <value>/root/hadoop/tmp</value>
                <description>Abase for other temporary directories.</description>
        </property>
        <property>
                <name>fs.default.name</name>
                <value>hdfs://master:9000</value>
        </property>
</configuration>

修改hdfs-site.xml

vim hdfs-site.xml

<configuration>
        <property>
                <name>dfs.name.dir</name>
                <value>/root/hadoop/dfs/name</value>
                <description>Path on the local filesystem where theNameNode stores the namespace and transactions logs persistently.</description>
        </property>
        <property>
                <name>dfs.data.dir</name>
                <value>/root/hadoop/dfs/data</value>
                <description>Comma separated list of paths on the localfilesystem of a DataNode where it should store its blocks.</description>
        </property>
        <property>
                <name>dfs.replication</name>
                <value>2</value>
        </property>
        <property>
                <name>dfs.permissions</name>
                <value>false</value>
                <description>need not permissions</description>
        </property>
</configuration>

修改mapred-site.xml

vim mapred-site.xml

<configuration>
        <property>
                <name>mapred.job.tracker</name>
                <value>master:49001</value>
        </property>
        <property>
                <name>mapred.local.dir</name>
                <value>/root/hadoop/var</value>
        </property>
        <property>
                <name>mapreduce.framework.name</name>
                <value>yarn</value>
        </property>
</configuration>

修改workers

vim workers

# 删除 localhost
slave1
slave2

修改yarn-site.xml

vim yarn-site.xml

<configuration>

<!-- Site specific YARN configuration properties -->
	<property>
		<name>yarn.resourcemanager.hostname</name>
		<value>master</value>
	</property>

	<property>
		<description>The address of the applications manager interface in the RM.</description>
		<name>yarn.resourcemanager.address</name>
		<value>${yarn.resourcemanager.hostname}:8032</value>
	</property>

	<property>
		<description>The address of the scheduler interface.</description>
		<name>yarn.resourcemanager.scheduler.address</name>
		<value>${yarn.resourcemanager.hostname}:8030</value>
	</property>

	<property>
		<description>The http address of the RM web application.</description>
		<name>yarn.resourcemanager.webapp.address</name>
		<value>${yarn.resourcemanager.hostname}:8088</value>
	</property>

	<property>
		<description>The https adddress of the RM web application.</description>
		<name>yarn.resourcemanager.webapp.https.address</name>
		<value>${yarn.resourcemanager.hostname}:8090</value>
	</property>

	<property>
		<name>yarn.resourcemanager.resource-tracker.address</name>
		<value>${yarn.resourcemanager.hostname}:8031</value>
	</property>

	<property>
		<description>The address of the RM admin interface.</description>
		<name>yarn.resourcemanager.admin.address</name>
		<value>${yarn.resourcemanager.hostname}:8033</value>
	</property>
	
	<property>
		<name>yarn.nodemanager.aux-services</name>
		<value>mapreduce_shuffle</value>
	</property>

	<property>
		<name>yarn.scheduler.maximum-allocation-mb</name>
		<value>1024</value>
		<discription>每个节点可用内存,单位MB,默认8182MB</discription>
	</property>

	<property>
		<name>yarn.nodemanager.vmem-pmem-ratio</name>
		<value>2.1</value>
	</property>

	<property>
		<name>yarn.nodemanager.resource.memory-mb</name>
		<value>1024</value>
	</property>

	<property>
		<name>yarn.nodemanager.vmem-check-enabled</name>
		<value>false</value>
	</property>
</configuration>

修改启动用户

• cd /opt/hadoop-3.2.2/sbin

start-dfs.sh
stop-dfs.sh
start-yarn.sh
stop-yarn.sh

start-dfs.sh

vim start-dfs.sh

# 文件上方添加以下内容
HDFS_DATANODE_USER=root
HADOOP_SECURE_DN_USER=root
HDFS_NAMENODE_USER=root
HDFS_SECONDARYNAMENODE_USER=root

stop-dfs.sh

vim stop-dfs.sh

# 文件上方添加以下内容
HDFS_DATANODE_USER=root
HADOOP_SECURE_DN_USER=root
HDFS_NAMENODE_USER=root
HDFS_SECONDARYNAMENODE_USER=root

修改hadoop-env.sh

export JAVA_HOME=/usr/local/java/jdk1.8.0_221

start-yarn.sh

vim start-yarn.sh

# 文件上方添加以下内容
YARN_RESOURCEMANAGER_USER=root
HADOOP_SECURE_DN_USER=root
YARN_NODEMANAGER_USER=root

stop-yarn.sh

vim stop-yarn.sh

# 文件上方添加以下内容
YARN_RESOURCEMANAGER_USER=root
HADOOP_SECURE_DN_USER=root
YARN_NODEMANAGER_USER=root

slave1, slave2 直接克隆 master 虚拟机

修改 slave1

修改 ip

vim /etc/sysconfig/network-scripts/ifcfg-ens33

# ip 更改为
IPADDR=192.168.22.132

# 重启网络
systemctl restart network

修改主机名

hostnamectl set-hostname slave1

# 修改 hosts
vim /etc/hosts

192.168.22.131 master
192.168.22.132 slave1
192.168.22.133 slave2

修改 slave2

修改 ip

vim /etc/sysconfig/network-scripts/ifcfg-ens33

# ip 更改为
IPADDR=192.168.22.133

# 重启网络
systemctl restart network

修改主机名

hostnamectl set-hostname slave2

# 修改 hosts
vim /etc/hosts

192.168.22.131 master
192.168.22.132 slave1
192.168.22.133 slave2

master hadoop格式化

# 格式化 hadoop
hadoop namenode -format

hadoop 启动

# 启动
start-all.sh

# 停止
stop-all.sh

访问

http://192.168.22.131:9870/dfshealth.html#tab-overview

http://192.168.22.131:8088/

Hive 安装

下载hive

https://dlcdn.apache.org/hive/hive-2.3.9/apache-hive-2.3.9-bin.tar.gz

配置 hive

# 解压文件
tar -xvf apache-hive-2.3.9-bin.tar.gz -C /opt/

配置环境变量

vim /etc/profile

export HIVE_HOME=/opt/apache-hive-2.3.9-bin

export PATH=$JAVA_HOME/bin:$HIVE_HOME/bin:$PATH

# 生效
source /etc/profile

设置hive执行权限

chmod u+x /opt/apache-hive-2.3.9-bin/bin/*

conf/hive-default.xml.template 复制成新的文件: hive-site.xml

  <property>
    <name>hive.exec.stagingdir</name>
    <value>/opt/apache-hive-2.3.9-bin/data/exec_logs</value>
    <description>Directory name that will be created inside table locations in order to support HDFS encryption. This is replaces ${hive.exec.scratchdir} for query results with the exception of read-only tables. In all cases ${hive.exec.scratchdir} is still used for other temporary files, such as job plans.</description>
  </property>
  <!--hive的临时数据目录，指定的位置在hdfs上的目录-->
  <property>
    <name>hive.exec.scratchdir</name>
    <value>/tmp/hive</value>
    <description>HDFS root scratch dir for Hive jobs which gets created with write all (733) permission. For each connecting user, an HDFS scratch dir: ${hive.exec.scratchdir}/&lt;username&gt; is created, with ${hive.scratch.dir.permission}.</description>
  </property>

  <!-- scratchdir 本地目录 -->
  <property>
    <name>hive.exec.local.scratchdir</name>
    <value>/opt/apache-hive-2.3.9-bin/data/scratch_dir</value>
    <description>Local scratch space for Hive jobs</description>
  </property>
  
  <!-- resources_dir 本地目录 -->
  <property>
    <name>hive.downloaded.resources.dir</name>
    <value>/opt/apache-hive-2.3.9-bin/data/resources_dir/</value>
    <description>Temporary local directory for added resources in the remote file system.</description>
  </property>

  <!--hive的临时数据目录，指定的位置在hdfs上的目录-->
  <property>
    <name>hive.metastore.warehouse.dir</name>
    <value>/user/hive/warehouse</value>
    <description>location of default database for the warehouse</description>
  </property>

  <!-- 数据库访问密码 -->
  <property>
    <name>javax.jdo.option.ConnectionPassword</name>
    <value>root</value>
    <description>password to use against metastore database</description>
  </property>

  <!-- 数据库连接地址配置 -->
  <property>
    <name>javax.jdo.option.ConnectionURL</name>
    <value>jdbc:mysql://127.0.0.1:3306/hive?serverTimezone=UTC&amp;useSSL=false&amp;allowPublicKeyRetrieval=true</value>
    <description>
      JDBC connect string for a JDBC metastore.
      To use SSL to encrypt/authenticate the connection, provide database-specific SSL flag in the connection URL.
      For example, jdbc:postgresql://myhost/db?ssl=true for postgres database.
    </description>
  </property>

  <!-- 自动创建全部 -->
  <!-- hive Required table missing : "DBS" in Catalog""Schema" 错误 -->
  <property>
    <name>datanucleus.schema.autoCreateAll</name>
    <value>true</value>
    <description>Auto creates necessary schema on a startup if one doesn't exist. Set this to false, after creating it once.To enable auto create also set hive.metastore.schema.verification=false. Auto creation is not recommended for production use cases, run schematool command instead.</description>
  </property>
  <!-- 解决 Caused by: MetaException(message:Version information not found in metastore. ) -->
  <property>
    <name>hive.metastore.schema.verification</name>
    <value>false</value>
    <description>
      Enforce metastore schema version consistency.
      True: Verify that version information stored in is compatible with one from Hive jars.  Also disable automatic
            schema migration attempt. Users are required to manually migrate schema after Hive upgrade which ensures
            proper metastore schema migration. (Default)
      False: Warn if the version information stored in metastore doesn't match with one from in Hive jars.
    </description>
  </property>

  <!-- 数据库驱动配置 -->
  <property>
    <name>javax.jdo.option.ConnectionDriverName</name>
    <value>com.mysql.cj.jdbc.Driver</value>
    <description>Driver class name for a JDBC metastore</description>
  </property>

  <!-- 数据库用户名 -->
  <property>
    <name>javax.jdo.option.ConnectionUserName</name>
    <value>root</value>
    <description>Username to use against metastore database</description>
  </property>

  <!-- operation_logs 本地目录 -->
  <property>
    <name>hive.server2.logging.operation.log.location</name>
    <value>/opt/apache-hive-2.3.9-bin/data/operation_logs</value>
    <description>Top level directory where operation logs are stored if logging functionality is enabled</description>
  </property>

conf/hive-env.sh.template 复制成新的文件: hive-env.sh

export HADOOP_HOME=/opt/hadoop-3.2.2

# Hive Configuration Directory can be controlled by:
export HIVE_CONF_DIR=/opt/apache-hive-2.3.9-bin/conf

# Folder containing extra libraries required for hive compilation/execution can be controlled by:
export HIVE_AUX_JARS_PATH=/opt/apache-hive-2.3.9-bin/lib

conf/hive-exec-log4j.properties.template 复制成新的文件: hive-exec-log4j2.properties

conf/hive-log4j.properties.template 复制成新的文件: hive-log4j2.properties

启动 Hive 服务

初始化 Hive 数据库

schematool -initSchema -dbType mysql

错误1解决

1. 删除/opt/apache-hive-2.3.9-bin/lib/guava-14.0.1.jar
2. rm -rf /opt/apache-hive-2.3.9-bin/lib/guava-14.0.1.jar
3. copy hadoop 下的版本 guava-27.0-jre.jar
4. cp /opt/hadoop-3.2.2/share/hadoop/common/lib/guava-27.0-jre.jar /opt/apache-hive-2.3.9-bin/lib/

错误2解决

连接数据库创建 hive database

初始化成功

进入hive

zookeeper 安装

下载zookeeper

https://dlcdn.apache.org/zookeeper/zookeeper-3.7.0/apache-zookeeper-3.7.0-bin.tar.gz

配置 zookeeper

# 解压文件
tar -xvf apache-zookeeper-3.7.0-bin.tar.gz -C /opt/

配置环境变量

vim /etc/profile

export ZOOKEEPER_HOME=/opt/apache-zookeeper-3.7.0-bin

export PATH=$JAVA_HOME/bin:$ZOOKEEPER_HOME/bin:$PATH

# 生效
source /etc/profile

conf/zoo_sample.cfg 复制成新的文件: zoo.cfg

在/opt/apache-zookeeper-3.7.0-bin目录下新建data，logs两个文件夹

# The number of milliseconds of each tick
tickTime=2000
# The number of ticks that the initial 
# synchronization phase can take
initLimit=10
# The number of ticks that can pass between 
# sending a request and getting an acknowledgement
syncLimit=5
# the directory where the snapshot is stored.
# do not use /tmp for storage, /tmp here is just 
# example sakes.
dataDir=/opt/apache-zookeeper-3.7.0-bin/data
dataLogDir=/opt/apache-zookeeper-3.7.0-bin/logs
# the port at which the clients will connect
clientPort=2181
# the maximum number of client connections.
# increase this if you need to handle more clients
#maxClientCnxns=60
#
# Be sure to read the maintenance section of the 
# administrator guide before turning on autopurge.
#
# http://zookeeper.apache.org/doc/current/zookeeperAdmin.html#sc_maintenance
#
# The number of snapshots to retain in dataDir
#autopurge.snapRetainCount=3
# Purge task interval in hours
# Set to "0" to disable auto purge feature
#autopurge.purgeInterval=1

## Metrics Providers
#
# https://prometheus.io Metrics Exporter
#metricsProvider.className=org.apache.zookeeper.metrics.prometheus.PrometheusMetricsProvider
#metricsProvider.httpPort=7000
#metricsProvider.exportJvmInfo=true

server.1=master:2888:3888
server.2=slave1:2888:3888
server.3=slave2:2888:3888

将配置好的zookeeper复制到另外两台服务器上

scp -r /opt/apache-zookeeper-3.7.0-bin slave1:/opt
scp -r /opt/apache-zookeeper-3.7.0-bin slave2:/opt

配置myid文件

分别在三台服务器的/data/zookeeper/zookeeper_cluster/zookeeper-3.4.9/data目录下新建myid文件，内容分别为master.1，slave1.2，slave2.3后面的数字
echo “1” > myid

配置zk集群启动停止脚本

#!/bin/bash  

usage="Usage: $0 (start|stop|status)"

if [ $# -lt 1 ]; then
  echo $usage
  exit 1
fi

behave=$1

iparray=(master slave1 slave2)

path="/opt/apache-zookeeper-3.7.0-bin"

echo "$behave zkServer cluster"

for ip in ${iparray[*]}  
do

    echo "ssh to $ip"

    ssh $ip "$path/bin/zkServer.sh $behave $path/conf/zoo.cfg"

    sleep 2s

done

exit 0

Hbase 安装

下载Hbase

https://dlcdn.apache.org/hbase/2.4.9/hbase-2.4.9-bin.tar.gz

配置 Hbase

# 解压文件
tar -xvf hbase-2.4.9-bin.tar.gz -C /opt/

配置环境变量

vim /etc/profile

export HBASE_HOME=/opt/hbase-2.4.9-bin

export PATH=$JAVA_HOME/bin:$HBASE_HOME/bin:$PATH

# 生效
source /etc/profile

conf/hbase-env.sh

export JAVA_HOME=/usr/local/java/jdk1.8.0_221

export HADOOP_HOME=/opt/hadoop-3.2.2

export HBASE_HOME=/opt/hbase-2.4.9

# 指定HBase是否使用HBase本身自带的Zookeeper
export HBASE_MANAGES_ZK=false

export HBASE_CLASSPATH=/opt/hbase-2.4.9/conf

export PATH=$JAVA_HOME/bin:$HADOOP_HOME/bin:$HBASE_HOME/bin:$PATH

conf/hbase-site.xml

<?xml version="1.0"?>
<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
<!--
/*
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
-->
<configuration>
  <!--
    The following properties are set for running HBase as a single process on a
    developer workstation. With this configuration, HBase is running in
    "stand-alone" mode and without a distributed file system. In this mode, and
    without further configuration, HBase and ZooKeeper data are stored on the
    local filesystem, in a path under the value configured for `hbase.tmp.dir`.
    This value is overridden from its default value of `/tmp` because many
    systems clean `/tmp` on a regular basis. Instead, it points to a path within
    this HBase installation directory.

    Running against the `LocalFileSystem`, as opposed to a distributed
    filesystem, runs the risk of data integrity issues and data loss. Normally
    HBase will refuse to run in such an environment. Setting
    `hbase.unsafe.stream.capability.enforce` to `false` overrides this behavior,
    permitting operation. This configuration is for the developer workstation
    only and __should not be used in production!__

    See also https://hbase.apache.org/book.html#standalone_dist
  -->
  <!-- 是否分布式部署 -->
  <property>
    <name>hbase.cluster.distributed</name>
    <value>true</value>
  </property>
  <!-- 指定hbase存放数据的HDFS目录，如果是分布式部署，要和Hadoop的core-site.xml中的fs.defaultFS一致-->
  <property>
    <name>hbase.rootdir</name>
    <value>hdfs://master:9000/hbase</value>
  </property>
  <property>
    <name>hbase.tmp.dir</name>
    <value>/opt/hbase-2.4.9/tmp</value>
  </property>
  <!-- 配置Zookeeper节点-->
  <property>
    <name>hbase.zookeeper.quorum</name>
    <value>master,slave1,slave2</value>
  </property>
  <!-- 设置zk集群端口，默认是2181，一定要和你的zk集群端口保持一致-->
  <property>
      <name>hbase.zookeeper.property.clientPort</name>
      <value>2181</value>
  </property>
  <!-- Hbase在zk上注册的数据信息，默认是/tmp,如果不修改，当系统重启的时候会删除/tmp目录 -->
  <property>
    <name>hbase.zookeeper.property.dataDir</name>
    <value>/opt/hbase-2.4.9/zkdata</value>
  </property>
  <property>
    <name>hbase.unsafe.stream.capability.enforce</name>
    <value>false</value>
  </property>
  <property>
    <name>hbase.master.info.port</name>
    <value>60010</value>
  </property>
</configuration>

conf/regionservers

slave1
slave2

配置好的zookeeper复制到另外两台服务器上

scp -r /opt/hbase-2.4.9 slave1:/opt
scp -r /opt/hbase-2.4.9 slave2:/opt

启动停止hbase

# 启动
start-hbase.sh

# 停止
stop-hbase.sh

# thrift
hbase-daemon.sh start thrift -p 9090

查看是否启动成功

访问页面

http://192.168.22.131:60010/master-status#userTables

Spark 安装

下载Spark

https://dlcdn.apache.org/spark/spark-3.2.1/spark-3.2.1-bin-hadoop3.2.tgz

配置 Spark

# 解压文件
tar -xvf spark-3.2.1-bin-hadoop3.2.tgz -C /opt/

配置环境变量

vim /etc/profile

export HBASE_HOME=/opt/spark-3.2.1-bin-hadoop3.2

export PATH=$JAVA_HOME/bin:$HBASE_HOME/bin:$PATH

# 生效
source /etc/profile

conf/spark-env.sh.template 复制为新文件：spark-env.sh

export JAVA_HOME=/usr/local/java/jdk1.8.0_221
export SPARK_MASTER_HOST=master
export SPARK_MASTER_PORT=7077

conf/workers.template 复制为新文件：workers

slave1
slave2

sbin/start-master.sh

# 修改页面访问端口，避免占用
SPARK_MASTER_WEBUI_PORT=8085

配置好的zookeeper复制到另外两台服务器上

scp -r /opt/spark-3.2.1-bin-hadoop3.2 slave1:/opt
scp -r /opt/spark-3.2.1-bin-hadoop3.2 slave2:/opt

启动停止Spark

# 启动
sbin/start-all.sh

# 停止
sbin/stop-all.sh