SS00022.PBpositions——|Hadoop&PB级离线数仓.v01|——|Griffin.v01|Griffin部署&spark&livy&ES|

一、安装概述

### --- 相关依赖：重点讲解 Griffin，不对依赖组件做过多讲解，所有组件均采用单机模式安装。

~~~     JDK (1.8 or later versions)
~~~     MySQL(version 5.6及以上)
~~~     Hadoop (2.6.0 or later)
~~~     Hive (version 2.x)
~~~     Maven
~~~     Spark (version 2.2.1)
~~~     Livy(livy-0.5.0-incubating)
~~~     ElasticSearch (5.0 or later versions)

### --- 依赖组件说明

~~~     Spark：计算批量、实时指标
~~~     Livy：为服务提供 RESTful API 调用 Apache Spark
~~~     ElasticSearch：存储指标数据
~~~     MySQL：服务元数据

二、Spark安装

### --- 下载并解压缩spark版本包

[root@hadoop02 ~]# ll /opt/yanqi/software/spark-2.2.1-bin-hadoop2.7.tgz 
-rw-r--r-- 1 root root 200934340 Aug 26  2020 /opt/yanqi/software/spark-2.2.1-bin-hadoop2.7.tgz

[root@hadoop02 ~]# cd /opt/yanqi/software/
[root@hadoop02 software]# tar -zxvf spark-2.2.1-bin-hadoop2.7.tgz -C ../servers/

[root@hadoop02 ~]# cd /opt/yanqi/servers/
[root@hadoop02 servers]# mv spark-2.2.1-bin-hadoop2.7/ spark-2.2.1

### --- 设置$SPARK_HOME环境变量

[root@hadoop02 ~]# vim /etc/profile
##SPARK_HOME
export SPARK_HOME=/opt/yanqi/servers/spark-2.2.1/
export PATH=$PATH:$SPARK_HOME/bin

~~~     # 使环境变量生效

[root@hadoop02 ~]# source /etc/profile

### --- 修改配置文件 $SPARK_HOME/conf/spark-defaults.conf

~~~     # 准备配置文件
[root@hadoop02 ~]# cd $SPARK_HOME/conf/

[root@hadoop02 conf]# cp spark-defaults.conf.template spark-defaults.conf
[root@hadoop02 conf]# cp spark-env.sh.template spark-env.sh

[root@hadoop02 ~]# vim $SPARK_HOME/conf/spark-defaults.conf
~~~ 修改配置参数：可以直接粘贴在里面
spark.master                            yarn
spark.eventLog.enabled                  true
spark.eventLog.dir                      hdfs://hadoop01:9000/spark/logs
spark.serializer                        org.apache.spark.serializer.KryoSerializer
spark.yarn.jars                         hdfs://hadoop01:9000/spark/spark_2.2.1_jars/*

### --- 拷贝 MySQL 驱动

[root@hadoop02 ~]# cp $HIVE_HOME/lib/mysql-connector-java-5.1.46.jar \
$SPARK_HOME/jars/
 
[root@hadoop02 ~]# ls $SPARK_HOME/jars/mysql-connector-java-5.1.46.jar 
/opt/yanqi/servers/spark-2.2.1//jars/mysql-connector-java-5.1.46.jar

### --- 将 Spark 的 jar 包上传到 hdfs://hadoop1:9000/spark/spark_2.2.1_jars/

[root@hadoop02 ~]# hdfs dfs -mkdir -p /spark/logs
[root@hadoop02 ~]# hdfs dfs -mkdir -p /spark/spark_2.2.1_jars/

[root@hadoop02 ~]# hdfs dfs -put /opt/yanqi/servers/spark-2.2.1/jars/*.jar /spark/spark_2.2.1_jars/

### --- 修改配置文件spark-env.sh

[root@hadoop02 ~]# vim $SPARK_HOME/conf/spark-env.sh
~~~ 输入参数
export JAVA_HOME=/opt/yanqi/servers/jdk1.8.0_231/
export HADOOP_HOME=/opt/yanqi/servers/hadoop-2.9.2/
export HADOOP_CONF_DIR=$HADOOP_HOME/etc/hadoop
export SPARK_DIST_CLASSPATH=$(hadoop classpath)
export YARN_CONF_DIR=$HADOOP_HOME/etc/hadoop

### --- yarn-site.xml 添加配置
~~~     yarn.nodemanager.vmem-check-enabled：是否检查虚拟内存。
~~~     修改所有节点，并重启yarn服务。
~~~     不添加该配配置启动spark-shell，有如下错误：Yarn application has already ended! It might have been killed or unable to launch application master.

[root@hadoop02 ~]# vim $HADOOP_HOME/etc/hadoop/yarn-site.xml
<!-- Spark参数配置：不去检查真实内存参数 -->
<property>
        <name>yarn.nodemanager.vmem-check-enabled</name>
        <value>false</value>
</property>

~~~     # 重启yarn服务

[root@hadoop01 ~]# stop-yarn.sh
[root@hadoop01 ~]# start-yarn.sh

### --- 测试spark：spark-shell

~~~     # 准备测试文件
[root@hadoop02 ~]# hdfs dfs -ls /wcinput/wc.txt        #  /wcinput/wc.txt : HDFS上的文件
-rw-r--r--   5 root supergroup        109 2020-06-20 13:58 /wcinput/wc.txt

~~~     # 进入spark-shell

[root@hadoop02 ~]# spark-shell
~~~ 输出参数
Welcome to
      ____              __
     / __/__  ___ _____/ /__
    _\ \/ _ \/ _ `/ __/  '_/
   /___/ .__/\_,_/_/ /_/\_\   version 2.2.1
      /_/

~~~     # 查看启动的进程

[root@hadoop02 ~]# jps
9422 SparkSubmit

~~~     # 打开该文件读取内容

scala> val lines = sc.textFile("/wcinput/wc.txt")
scala> lines.flatMap(_.split(" ")).map((_,1)).reduceByKey(_+_).collect()
~~~ 输出参数
res1: Array[(String, Int)] = Array((#在文件中输入如下内容,1), (yanqi,3), (mapreduce,3), (yarn,2), (hadoop,2), (hdfs,1))

三、Livy安装

### --- 下载版本包并解压缩

[root@hadoop02 ~]# ls /opt/yanqi/software/livy-0.5.0-incubating-bin.zip 
/opt/yanqi/software/livy-0.5.0-incubating-bin.zip

[root@hadoop02 ~]# cd /opt/yanqi/software/
[root@hadoop02 software]# unzip livy-0.5.0-incubating-bin.zip

[root@hadoop02 software]# mv livy-0.5.0-incubating-bin ../servers/livy-0.5.0

### --- 设置环境变量 $LIVY_HOME

[root@hadoop02 ~]# vim /etc/profile
## LIVY_HOME
export LIVY_HOME=/opt/yanqi/servers/livy-0.5.0
export PATH=$PATH:$LIVY_HOME/bin

[root@hadoop02 ~]# source /etc/profile

### --- 修改配置文件 conf/livy.conf

~~~     # 准备配置文件
[root@hadoop02 ~]# cd $LIVY_HOME/conf
 
[root@hadoop02 conf]# cp livy-env.sh.template livy-env.sh
[root@hadoop02 conf]# cp livy.conf.template livy.conf

[root@hadoop02 ~]# vim $LIVY_HOME/conf/livy.conf
~~~ 修改配置参数
livy.server.host = 127.0.0.1
livy.spark.master = yarn
livy.spark.deployMode = cluster
livy.repl.enable-hive-context = true

### --- 修改配置文件 conf/livy-env.sh

[root@hadoop02 ~]# vim $LIVY_HOME/conf/livy-env.sh
~~~ 修改配置参数
export SPARK_HOME=/opt/yanqi/servers/spark-2.2.1
export HADOOP_HOME=/opt/yanqi/servers/hadoop-2.9.2
export HADOOP_CONF_DIR=$HADOOP_HOME/etc/hadoop

### --- 启动服务

[root@hadoop02 ~]# cd $LIVY_HOME/
[root@hadoop02 livy-0.5.0]# mkdir logs
[root@hadoop02 livy-0.5.0]# nohup bin/livy-server &
~~~     # OR
[root@hadoop02 ~]# nohup $LIVY_HOME/bin/livy-server &

~~~     # 查看启动的服务

[root@hadoop02 ~]# jps
7897 SparkSubmit
8139 LivyServer

四、ES安装

### --- 下载并解压缩ES版本包

[root@hadoop02 ~]# ls /opt/yanqi/software/elasticsearch-5.6.0.tar.gz 
/opt/yanqi/software/elasticsearch-5.6.0.tar.gz
[root@hadoop02 software]# tar -zxvf elasticsearch-5.6.0.tar.gz -C ../servers/

### --- 创建 elasticsearch用户组 及 elasticsearch 用户。

~~~     # 不能使用root用户启动ES程序，需要创建单独的用户去启动ES 服务；
~~~     创建用户组
[root@hadoop02 ~]# groupadd elasticsearch
~~~     创建用户
[root@hadoop02 ~]# useradd elasticsearch -g elasticsearch
~~~     修改安装目录的宿主
[root@hadoop02 ~]# cd /opt/yanqi/servers/
[root@hadoop02 servers]# chown -R elasticsearch:elasticsearch elasticsearch-5.6.0/

### --- 修改linux系统文件 /etc/security/limits.conf

[root@hadoop02 ~]# vim /etc/security/limits.conf
~~~ 修改配置参数
elasticsearch hard nofile 1000000
elasticsearch soft nofile 1000000
* soft nproc 4096
* hard nproc 4096

### --- 修改系统文件 /etc/sysctl.conf

~~~     # 修改配置参数
[root@hadoop02 ~]# vim /etc/sysctl.conf
~~~ 文件末尾增加：
vm.max_map_count=262144
~~~     # 执行以下命令，修改才能生效
[root@hadoop02 ~]# sysctl -p
vm.max_map_count = 262144

### --- 修改es配置文件

~~~     # 修改es的配置参数
[root@hadoop02 ~]# vim /opt/yanqi/servers/elasticsearch-5.6.0/config/elasticsearch.yml
~~~ 修改配置参数
network.host: 0.0.0.0

~~~     # jvm内存的分配，原来都是2g，修改为1g

~~~     # 修改jvm内存参数
[root@hadoop02 ~]# vim /opt/yanqi/servers/elasticsearch-5.6.0/config/jvm.options
~~~ 修改内存参数
-Xms1g
-Xmx1g

### --- 启动ES服务
### --- 通过浏览器访问es服务：http://hadoop02:9200/

~~~     # 到ES安装目录下，执行命令(-d表示后台启动)
[root@hadoop02 ~]# su elasticsearch
 
[elasticsearch@hadoop02 ~]$ cd /opt/yanqi/servers/elasticsearch-5.6.0/
[elasticsearch@hadoop02 elasticsearch-5.6.0]$ bin/elasticsearch -d

### --- 在ES里创建griffin索引

~~~     # hadoop02 为 ES 服务所在节点
[elasticsearch@hadoop02 ~]$  curl -XPUT http://hadoop02:9200/griffin -d '
{
    "aliases": {},
        "mappings": {
            "accuracy": {
                "properties": {
                    "name": {
                        "fields": {
                            "keyword": {
                                "ignore_above": 256,
                                    "type": "keyword"
                            }
                        },
                            "type": "text"
                    },
                        "tmst": {
                            "type": "date"
                        }
                }
            }
        },
            "settings": {
                "index": {
                    "number_of_replicas": "2",
                        "number_of_shards": "5"
                }
            }
}
'
~~~ 输出参数
{"acknowledged":true,"shards_acknowledged":true,"index":"griffin"}